tcp_input.c 170.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		Implementation of the Transmission Control Protocol(TCP).
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Mark Evans, <evansmp@uhura.aston.ac.uk>
 *		Corey Minyard <wf-rch!minyard@relay.EU.net>
 *		Florian La Roche, <flla@stud.uni-sb.de>
 *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
 *		Linus Torvalds, <torvalds@cs.helsinki.fi>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Matthew Dillon, <dillon@apollo.west.oic.com>
 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *		Jorge Cwik, <jorge@laser.satlink.net>
 */

/*
 * Changes:
 *		Pedro Roque	:	Fast Retransmit/Recovery.
 *					Two receive queues.
 *					Retransmit queue handled by TCP.
 *					Better retransmit timer handling.
 *					New congestion avoidance.
 *					Header prediction.
 *					Variable renaming.
 *
 *		Eric		:	Fast Retransmit.
 *		Randy Scott	:	MSS option defines.
 *		Eric Schenk	:	Fixes to slow start algorithm.
 *		Eric Schenk	:	Yet another double ACK bug.
 *		Eric Schenk	:	Delayed ACK bug fixes.
 *		Eric Schenk	:	Floyd style fast retrans war avoidance.
 *		David S. Miller	:	Don't allow zero congestion window.
 *		Eric Schenk	:	Fix retransmitter so that it sends
 *					next packet on ack of previous packet.
 *		Andi Kleen	:	Moved open_request checking here
 *					and process RSTs for open_requests.
 *		Andi Kleen	:	Better prune_queue, and other fixes.
S
Stephen Hemminger 已提交
43
 *		Andrey Savochkin:	Fix RTT measurements in the presence of
L
Linus Torvalds 已提交
44 45 46 47 48 49 50
 *					timestamps.
 *		Andrey Savochkin:	Check sequence numbers correctly when
 *					removing SACKs due to in sequence incoming
 *					data segments.
 *		Andi Kleen:		Make sure we never ack data there is not
 *					enough room for. Also make this condition
 *					a fatal error if it might still happen.
51
 *		Andi Kleen:		Add tcp_measure_rcv_mss to make
L
Linus Torvalds 已提交
52
 *					connections with MSS<min(MTU,ann. MSS)
53
 *					work without delayed acks.
L
Linus Torvalds 已提交
54 55 56 57 58 59 60 61 62 63
 *		Andi Kleen:		Process packets with PSH set in the
 *					fast path.
 *		J Hadi Salim:		ECN support
 *	 	Andrei Gurtov,
 *		Pasi Sarolahti,
 *		Panu Kuhlberg:		Experimental audit of TCP (re)transmission
 *					engine. Lots of bugs are found.
 *		Pasi Sarolahti:		F-RTO for dealing with spurious RTOs
 */

64 65
#define pr_fmt(fmt) "TCP: " fmt

L
Linus Torvalds 已提交
66
#include <linux/mm.h>
67
#include <linux/slab.h>
L
Linus Torvalds 已提交
68 69
#include <linux/module.h>
#include <linux/sysctl.h>
70
#include <linux/kernel.h>
71
#include <linux/prefetch.h>
72
#include <net/dst.h>
L
Linus Torvalds 已提交
73 74 75 76
#include <net/tcp.h>
#include <net/inet_common.h>
#include <linux/ipsec.h>
#include <asm/unaligned.h>
77
#include <linux/errqueue.h>
L
Linus Torvalds 已提交
78

79 80 81 82 83
int sysctl_tcp_timestamps __read_mostly = 1;
int sysctl_tcp_window_scaling __read_mostly = 1;
int sysctl_tcp_sack __read_mostly = 1;
int sysctl_tcp_fack __read_mostly = 1;
int sysctl_tcp_reordering __read_mostly = TCP_FASTRETRANS_THRESH;
84
int sysctl_tcp_max_reordering __read_mostly = 300;
E
Eric Dumazet 已提交
85
EXPORT_SYMBOL(sysctl_tcp_reordering);
86 87
int sysctl_tcp_dsack __read_mostly = 1;
int sysctl_tcp_app_win __read_mostly = 31;
88
int sysctl_tcp_adv_win_scale __read_mostly = 1;
E
Eric Dumazet 已提交
89
EXPORT_SYMBOL(sysctl_tcp_adv_win_scale);
L
Linus Torvalds 已提交
90

E
Eric Dumazet 已提交
91 92 93
/* rfc5961 challenge ack rate limiting */
int sysctl_tcp_challenge_ack_limit = 100;

94 95 96
int sysctl_tcp_stdurg __read_mostly;
int sysctl_tcp_rfc1337 __read_mostly;
int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
97
int sysctl_tcp_frto __read_mostly = 2;
L
Linus Torvalds 已提交
98

A
Andreas Petlund 已提交
99 100
int sysctl_tcp_thin_dupack __read_mostly;

101
int sysctl_tcp_moderate_rcvbuf __read_mostly = 1;
N
Nandita Dukkipati 已提交
102
int sysctl_tcp_early_retrans __read_mostly = 3;
L
Linus Torvalds 已提交
103 104 105 106 107 108 109 110 111

#define FLAG_DATA		0x01 /* Incoming frame contained data.		*/
#define FLAG_WIN_UPDATE		0x02 /* Incoming ACK was a window update.	*/
#define FLAG_DATA_ACKED		0x04 /* This ACK acknowledged new data.		*/
#define FLAG_RETRANS_DATA_ACKED	0x08 /* "" "" some of which was retransmitted.	*/
#define FLAG_SYN_ACKED		0x10 /* This ACK acknowledged SYN.		*/
#define FLAG_DATA_SACKED	0x20 /* New SACK.				*/
#define FLAG_ECE		0x40 /* ECE in this ACK				*/
#define FLAG_SLOWPATH		0x100 /* Do not skip RFC checks for window update.*/
Y
Yuchung Cheng 已提交
112
#define FLAG_ORIG_SACK_ACKED	0x200 /* Never retransmitted data are (s)acked	*/
113
#define FLAG_SND_UNA_ADVANCED	0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
114
#define FLAG_DSACKING_ACK	0x800 /* SACK blocks contained D-SACK info */
115
#define FLAG_SACK_RENEGING	0x2000 /* snd_una advanced to a sacked seq */
116
#define FLAG_UPDATE_TS_RECENT	0x4000 /* tcp_replace_ts_recent() */
L
Linus Torvalds 已提交
117 118 119 120 121 122 123

#define FLAG_ACKED		(FLAG_DATA_ACKED|FLAG_SYN_ACKED)
#define FLAG_NOT_DUP		(FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED)
#define FLAG_CA_ALERT		(FLAG_DATA_SACKED|FLAG_ECE)
#define FLAG_FORWARD_PROGRESS	(FLAG_ACKED|FLAG_DATA_SACKED)

#define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH)
124
#define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH))
L
Linus Torvalds 已提交
125

126
/* Adapt the MSS value used to make delayed ack decision to the
L
Linus Torvalds 已提交
127
 * real world.
128
 */
129
static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
130
{
131
	struct inet_connection_sock *icsk = inet_csk(sk);
132
	const unsigned int lss = icsk->icsk_ack.last_seg_size;
133
	unsigned int len;
L
Linus Torvalds 已提交
134

135
	icsk->icsk_ack.last_seg_size = 0;
L
Linus Torvalds 已提交
136 137 138 139

	/* skb->len may jitter because of SACKs, even if peer
	 * sends good full-sized frames.
	 */
140
	len = skb_shinfo(skb)->gso_size ? : skb->len;
141 142
	if (len >= icsk->icsk_ack.rcv_mss) {
		icsk->icsk_ack.rcv_mss = len;
L
Linus Torvalds 已提交
143 144 145 146 147 148
	} else {
		/* Otherwise, we make more careful check taking into account,
		 * that SACKs block is variable.
		 *
		 * "len" is invariant segment length, including TCP header.
		 */
149
		len += skb->data - skb_transport_header(skb);
150
		if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) ||
L
Linus Torvalds 已提交
151 152 153 154 155 156
		    /* If PSH is not set, packet should be
		     * full sized, provided peer TCP is not badly broken.
		     * This observation (if it is correct 8)) allows
		     * to handle super-low mtu links fairly.
		     */
		    (len >= TCP_MIN_MSS + sizeof(struct tcphdr) &&
157
		     !(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) {
L
Linus Torvalds 已提交
158 159 160 161
			/* Subtract also invariant (if peer is RFC compliant),
			 * tcp header plus fixed timestamp option length.
			 * Resulting "len" is MSS free of SACK jitter.
			 */
162 163
			len -= tcp_sk(sk)->tcp_header_len;
			icsk->icsk_ack.last_seg_size = len;
L
Linus Torvalds 已提交
164
			if (len == lss) {
165
				icsk->icsk_ack.rcv_mss = len;
L
Linus Torvalds 已提交
166 167 168
				return;
			}
		}
169 170
		if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)
			icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2;
171
		icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
L
Linus Torvalds 已提交
172 173 174
	}
}

175
static void tcp_incr_quickack(struct sock *sk)
L
Linus Torvalds 已提交
176
{
177
	struct inet_connection_sock *icsk = inet_csk(sk);
178
	unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss);
L
Linus Torvalds 已提交
179

180 181
	if (quickacks == 0)
		quickacks = 2;
182 183
	if (quickacks > icsk->icsk_ack.quick)
		icsk->icsk_ack.quick = min(quickacks, TCP_MAX_QUICKACKS);
L
Linus Torvalds 已提交
184 185
}

186
static void tcp_enter_quickack_mode(struct sock *sk)
L
Linus Torvalds 已提交
187
{
188 189 190 191
	struct inet_connection_sock *icsk = inet_csk(sk);
	tcp_incr_quickack(sk);
	icsk->icsk_ack.pingpong = 0;
	icsk->icsk_ack.ato = TCP_ATO_MIN;
L
Linus Torvalds 已提交
192 193 194 195 196 197
}

/* Send ACKs quickly, if "quick" count is not exhausted
 * and the session is not interactive.
 */

E
Eric Dumazet 已提交
198
static inline bool tcp_in_quickack_mode(const struct sock *sk)
L
Linus Torvalds 已提交
199
{
200
	const struct inet_connection_sock *icsk = inet_csk(sk);
E
Eric Dumazet 已提交
201

202
	return icsk->icsk_ack.quick && !icsk->icsk_ack.pingpong;
L
Linus Torvalds 已提交
203 204
}

205
static void tcp_ecn_queue_cwr(struct tcp_sock *tp)
206
{
207
	if (tp->ecn_flags & TCP_ECN_OK)
208 209 210
		tp->ecn_flags |= TCP_ECN_QUEUE_CWR;
}

211
static void tcp_ecn_accept_cwr(struct tcp_sock *tp, const struct sk_buff *skb)
212 213 214 215 216
{
	if (tcp_hdr(skb)->cwr)
		tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
}

217
static void tcp_ecn_withdraw_cwr(struct tcp_sock *tp)
218 219 220 221
{
	tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
}

222
static void __tcp_ecn_check_ce(struct tcp_sock *tp, const struct sk_buff *skb)
223
{
224
	switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) {
225
	case INET_ECN_NOT_ECT:
226
		/* Funny extension: if ECT is not set on a segment,
227 228 229 230
		 * and we already seen ECT on a previous segment,
		 * it is probably a retransmit.
		 */
		if (tp->ecn_flags & TCP_ECN_SEEN)
231
			tcp_enter_quickack_mode((struct sock *)tp);
232 233
		break;
	case INET_ECN_CE:
234 235 236
		if (tcp_ca_needs_ecn((struct sock *)tp))
			tcp_ca_event((struct sock *)tp, CA_EVENT_ECN_IS_CE);

237 238 239 240 241
		if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) {
			/* Better not delay acks, sender can have a very low cwnd */
			tcp_enter_quickack_mode((struct sock *)tp);
			tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
		}
242 243
		tp->ecn_flags |= TCP_ECN_SEEN;
		break;
244
	default:
245 246
		if (tcp_ca_needs_ecn((struct sock *)tp))
			tcp_ca_event((struct sock *)tp, CA_EVENT_ECN_NO_CE);
247
		tp->ecn_flags |= TCP_ECN_SEEN;
248
		break;
249 250 251
	}
}

252 253 254 255 256 257 258
static void tcp_ecn_check_ce(struct tcp_sock *tp, const struct sk_buff *skb)
{
	if (tp->ecn_flags & TCP_ECN_OK)
		__tcp_ecn_check_ce(tp, skb);
}

static void tcp_ecn_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th)
259
{
260
	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || th->cwr))
261 262 263
		tp->ecn_flags &= ~TCP_ECN_OK;
}

264
static void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th)
265
{
266
	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || !th->cwr))
267 268 269
		tp->ecn_flags &= ~TCP_ECN_OK;
}

270
static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr *th)
271
{
272
	if (th->ece && !th->syn && (tp->ecn_flags & TCP_ECN_OK))
E
Eric Dumazet 已提交
273 274
		return true;
	return false;
275 276
}

L
Linus Torvalds 已提交
277 278 279 280 281
/* Buffer size and advertised window tuning.
 *
 * 1. Tuning sk->sk_sndbuf, when connection enters established state.
 */

E
Eric Dumazet 已提交
282
static void tcp_sndbuf_expand(struct sock *sk)
L
Linus Torvalds 已提交
283
{
E
Eric Dumazet 已提交
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
	const struct tcp_sock *tp = tcp_sk(sk);
	int sndmem, per_mss;
	u32 nr_segs;

	/* Worst case is non GSO/TSO : each frame consumes one skb
	 * and skb->head is kmalloced using power of two area of memory
	 */
	per_mss = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) +
		  MAX_TCP_HEADER +
		  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

	per_mss = roundup_pow_of_two(per_mss) +
		  SKB_DATA_ALIGN(sizeof(struct sk_buff));

	nr_segs = max_t(u32, TCP_INIT_CWND, tp->snd_cwnd);
	nr_segs = max_t(u32, nr_segs, tp->reordering + 1);

	/* Fast Recovery (RFC 5681 3.2) :
	 * Cubic needs 1.7 factor, rounded to 2 to include
	 * extra cushion (application might react slowly to POLLOUT)
	 */
	sndmem = 2 * nr_segs * per_mss;
L
Linus Torvalds 已提交
306

307 308
	if (sk->sk_sndbuf < sndmem)
		sk->sk_sndbuf = min(sndmem, sysctl_tcp_wmem[2]);
L
Linus Torvalds 已提交
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
}

/* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
 *
 * All tcp_full_space() is split to two parts: "network" buffer, allocated
 * forward and advertised in receiver window (tp->rcv_wnd) and
 * "application buffer", required to isolate scheduling/application
 * latencies from network.
 * window_clamp is maximal advertised window. It can be less than
 * tcp_full_space(), in this case tcp_full_space() - window_clamp
 * is reserved for "application" buffer. The less window_clamp is
 * the smoother our behaviour from viewpoint of network, but the lower
 * throughput and the higher sensitivity of the connection to losses. 8)
 *
 * rcv_ssthresh is more strict window_clamp used at "slow start"
 * phase to predict further behaviour of this connection.
 * It is used for two goals:
 * - to enforce header prediction at sender, even when application
 *   requires some significant "application buffer". It is check #1.
 * - to prevent pruning of receive queue because of misprediction
 *   of receiver window. Check #2.
 *
 * The scheme does not work when sender sends good segments opening
S
Stephen Hemminger 已提交
332
 * window and then starts to feed us spaghetti. But it should work
L
Linus Torvalds 已提交
333 334 335 336
 * in common situations. Otherwise, we have to rely on queue collapsing.
 */

/* Slow part of check#2. */
337
static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
338
{
339
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
340
	/* Optimize this! */
341 342
	int truesize = tcp_win_from_space(skb->truesize) >> 1;
	int window = tcp_win_from_space(sysctl_tcp_rmem[2]) >> 1;
L
Linus Torvalds 已提交
343 344 345

	while (tp->rcv_ssthresh <= window) {
		if (truesize <= skb->len)
346
			return 2 * inet_csk(sk)->icsk_ack.rcv_mss;
L
Linus Torvalds 已提交
347 348 349 350 351 352 353

		truesize >>= 1;
		window >>= 1;
	}
	return 0;
}

354
static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
355
{
356 357
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
358 359 360
	/* Check #1 */
	if (tp->rcv_ssthresh < tp->window_clamp &&
	    (int)tp->rcv_ssthresh < tcp_space(sk) &&
361
	    !sk_under_memory_pressure(sk)) {
L
Linus Torvalds 已提交
362 363 364 365 366 367
		int incr;

		/* Check #2. Increase window, if skb with such overhead
		 * will fit to rcvbuf in future.
		 */
		if (tcp_win_from_space(skb->truesize) <= skb->len)
368
			incr = 2 * tp->advmss;
L
Linus Torvalds 已提交
369
		else
370
			incr = __tcp_grow_window(sk, skb);
L
Linus Torvalds 已提交
371 372

		if (incr) {
373
			incr = max_t(int, incr, 2 * skb->len);
374 375
			tp->rcv_ssthresh = min(tp->rcv_ssthresh + incr,
					       tp->window_clamp);
376
			inet_csk(sk)->icsk_ack.quick |= 1;
L
Linus Torvalds 已提交
377 378 379 380 381 382 383
		}
	}
}

/* 3. Tuning rcvbuf, when connection enters established state. */
static void tcp_fixup_rcvbuf(struct sock *sk)
{
384 385
	u32 mss = tcp_sk(sk)->advmss;
	int rcvmem;
L
Linus Torvalds 已提交
386

387 388
	rcvmem = 2 * SKB_TRUESIZE(mss + MAX_TCP_HEADER) *
		 tcp_default_init_rwnd(mss);
389

E
Eric Dumazet 已提交
390 391 392 393 394 395
	/* Dynamic Right Sizing (DRS) has 2 to 3 RTT latency
	 * Allow enough cushion so that sender is not limited by our window
	 */
	if (sysctl_tcp_moderate_rcvbuf)
		rcvmem <<= 2;

396 397
	if (sk->sk_rcvbuf < rcvmem)
		sk->sk_rcvbuf = min(rcvmem, sysctl_tcp_rmem[2]);
L
Linus Torvalds 已提交
398 399
}

S
Stephen Hemminger 已提交
400
/* 4. Try to fixup all. It is made immediately after connection enters
L
Linus Torvalds 已提交
401 402
 *    established state.
 */
403
void tcp_init_buffer_space(struct sock *sk)
L
Linus Torvalds 已提交
404 405 406 407 408 409 410
{
	struct tcp_sock *tp = tcp_sk(sk);
	int maxwin;

	if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK))
		tcp_fixup_rcvbuf(sk);
	if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK))
E
Eric Dumazet 已提交
411
		tcp_sndbuf_expand(sk);
L
Linus Torvalds 已提交
412 413

	tp->rcvq_space.space = tp->rcv_wnd;
E
Eric Dumazet 已提交
414 415
	tp->rcvq_space.time = tcp_time_stamp;
	tp->rcvq_space.seq = tp->copied_seq;
L
Linus Torvalds 已提交
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438

	maxwin = tcp_full_space(sk);

	if (tp->window_clamp >= maxwin) {
		tp->window_clamp = maxwin;

		if (sysctl_tcp_app_win && maxwin > 4 * tp->advmss)
			tp->window_clamp = max(maxwin -
					       (maxwin >> sysctl_tcp_app_win),
					       4 * tp->advmss);
	}

	/* Force reservation of one segment. */
	if (sysctl_tcp_app_win &&
	    tp->window_clamp > 2 * tp->advmss &&
	    tp->window_clamp + tp->advmss > maxwin)
		tp->window_clamp = max(2 * tp->advmss, maxwin - tp->advmss);

	tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp);
	tp->snd_cwnd_stamp = tcp_time_stamp;
}

/* 5. Recalculate window clamp after socket hit its memory bounds. */
439
static void tcp_clamp_window(struct sock *sk)
L
Linus Torvalds 已提交
440
{
441
	struct tcp_sock *tp = tcp_sk(sk);
442
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
443

444
	icsk->icsk_ack.quick = 0;
L
Linus Torvalds 已提交
445

446 447
	if (sk->sk_rcvbuf < sysctl_tcp_rmem[2] &&
	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) &&
448 449
	    !sk_under_memory_pressure(sk) &&
	    sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)) {
450 451
		sk->sk_rcvbuf = min(atomic_read(&sk->sk_rmem_alloc),
				    sysctl_tcp_rmem[2]);
L
Linus Torvalds 已提交
452
	}
453
	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
454
		tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss);
L
Linus Torvalds 已提交
455 456
}

S
Stephen Hemminger 已提交
457 458 459 460 461 462 463 464 465
/* Initialize RCV_MSS value.
 * RCV_MSS is an our guess about MSS used by the peer.
 * We haven't any direct information about the MSS.
 * It's better to underestimate the RCV_MSS rather than overestimate.
 * Overestimations make us ACKing less frequently than needed.
 * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss().
 */
void tcp_initialize_rcv_mss(struct sock *sk)
{
466
	const struct tcp_sock *tp = tcp_sk(sk);
S
Stephen Hemminger 已提交
467 468
	unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache);

469
	hint = min(hint, tp->rcv_wnd / 2);
470
	hint = min(hint, TCP_MSS_DEFAULT);
S
Stephen Hemminger 已提交
471 472 473 474
	hint = max(hint, TCP_MIN_MSS);

	inet_csk(sk)->icsk_ack.rcv_mss = hint;
}
E
Eric Dumazet 已提交
475
EXPORT_SYMBOL(tcp_initialize_rcv_mss);
S
Stephen Hemminger 已提交
476

L
Linus Torvalds 已提交
477 478 479 480
/* Receiver "autotuning" code.
 *
 * The algorithm for RTT estimation w/o timestamps is based on
 * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL.
481
 * <http://public.lanl.gov/radiant/pubs.html#DRS>
L
Linus Torvalds 已提交
482 483
 *
 * More detail on this code can be found at
484
 * <http://staff.psc.edu/jheffner/>,
L
Linus Torvalds 已提交
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
 * though this reference is out of date.  A new paper
 * is pending.
 */
static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)
{
	u32 new_sample = tp->rcv_rtt_est.rtt;
	long m = sample;

	if (m == 0)
		m = 1;

	if (new_sample != 0) {
		/* If we sample in larger samples in the non-timestamp
		 * case, we could grossly overestimate the RTT especially
		 * with chatty applications or bulk transfer apps which
		 * are stalled on filesystem I/O.
		 *
		 * Also, since we are only going for a minimum in the
S
Stephen Hemminger 已提交
503
		 * non-timestamp case, we do not smooth things out
S
Stephen Hemminger 已提交
504
		 * else with timestamps disabled convergence takes too
L
Linus Torvalds 已提交
505 506 507 508 509
		 * long.
		 */
		if (!win_dep) {
			m -= (new_sample >> 3);
			new_sample += m;
510 511 512 513 514
		} else {
			m <<= 3;
			if (m < new_sample)
				new_sample = m;
		}
L
Linus Torvalds 已提交
515
	} else {
S
Stephen Hemminger 已提交
516
		/* No previous measure. */
L
Linus Torvalds 已提交
517 518 519 520 521 522 523 524 525 526 527 528 529
		new_sample = m << 3;
	}

	if (tp->rcv_rtt_est.rtt != new_sample)
		tp->rcv_rtt_est.rtt = new_sample;
}

static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp)
{
	if (tp->rcv_rtt_est.time == 0)
		goto new_measure;
	if (before(tp->rcv_nxt, tp->rcv_rtt_est.seq))
		return;
530
	tcp_rcv_rtt_update(tp, tcp_time_stamp - tp->rcv_rtt_est.time, 1);
L
Linus Torvalds 已提交
531 532 533 534 535 536

new_measure:
	tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd;
	tp->rcv_rtt_est.time = tcp_time_stamp;
}

537 538
static inline void tcp_rcv_rtt_measure_ts(struct sock *sk,
					  const struct sk_buff *skb)
L
Linus Torvalds 已提交
539
{
540
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
541 542
	if (tp->rx_opt.rcv_tsecr &&
	    (TCP_SKB_CB(skb)->end_seq -
543
	     TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss))
L
Linus Torvalds 已提交
544 545 546 547 548 549 550 551 552 553 554
		tcp_rcv_rtt_update(tp, tcp_time_stamp - tp->rx_opt.rcv_tsecr, 0);
}

/*
 * This function should be called every time data is copied to user space.
 * It calculates the appropriate TCP receive buffer space.
 */
void tcp_rcv_space_adjust(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	int time;
E
Eric Dumazet 已提交
555
	int copied;
556

L
Linus Torvalds 已提交
557
	time = tcp_time_stamp - tp->rcvq_space.time;
558
	if (time < (tp->rcv_rtt_est.rtt >> 3) || tp->rcv_rtt_est.rtt == 0)
L
Linus Torvalds 已提交
559
		return;
560

E
Eric Dumazet 已提交
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
	/* Number of bytes copied to user in last RTT */
	copied = tp->copied_seq - tp->rcvq_space.seq;
	if (copied <= tp->rcvq_space.space)
		goto new_measure;

	/* A bit of theory :
	 * copied = bytes received in previous RTT, our base window
	 * To cope with packet losses, we need a 2x factor
	 * To cope with slow start, and sender growing its cwin by 100 %
	 * every RTT, we need a 4x factor, because the ACK we are sending
	 * now is for the next RTT, not the current one :
	 * <prev RTT . ><current RTT .. ><next RTT .... >
	 */

	if (sysctl_tcp_moderate_rcvbuf &&
	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
		int rcvwin, rcvmem, rcvbuf;
L
Linus Torvalds 已提交
578

E
Eric Dumazet 已提交
579 580 581 582
		/* minimal window to cope with packet losses, assuming
		 * steady state. Add some cushion because of small variations.
		 */
		rcvwin = (copied << 1) + 16 * tp->advmss;
L
Linus Torvalds 已提交
583

E
Eric Dumazet 已提交
584 585 586 587 588 589 590 591 592 593 594 595 596
		/* If rate increased by 25%,
		 *	assume slow start, rcvwin = 3 * copied
		 * If rate increased by 50%,
		 *	assume sender can use 2x growth, rcvwin = 4 * copied
		 */
		if (copied >=
		    tp->rcvq_space.space + (tp->rcvq_space.space >> 2)) {
			if (copied >=
			    tp->rcvq_space.space + (tp->rcvq_space.space >> 1))
				rcvwin <<= 1;
			else
				rcvwin += (rcvwin >> 1);
		}
L
Linus Torvalds 已提交
597

E
Eric Dumazet 已提交
598 599 600
		rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER);
		while (tcp_win_from_space(rcvmem) < tp->advmss)
			rcvmem += 128;
L
Linus Torvalds 已提交
601

E
Eric Dumazet 已提交
602 603 604
		rcvbuf = min(rcvwin / tp->advmss * rcvmem, sysctl_tcp_rmem[2]);
		if (rcvbuf > sk->sk_rcvbuf) {
			sk->sk_rcvbuf = rcvbuf;
L
Linus Torvalds 已提交
605

E
Eric Dumazet 已提交
606 607
			/* Make the window clamp follow along.  */
			tp->window_clamp = rcvwin;
L
Linus Torvalds 已提交
608 609
		}
	}
E
Eric Dumazet 已提交
610
	tp->rcvq_space.space = copied;
611

L
Linus Torvalds 已提交
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
new_measure:
	tp->rcvq_space.seq = tp->copied_seq;
	tp->rcvq_space.time = tcp_time_stamp;
}

/* There is something which you must keep in mind when you analyze the
 * behavior of the tp->ato delayed ack timeout interval.  When a
 * connection starts up, we want to ack as quickly as possible.  The
 * problem is that "good" TCP's do slow start at the beginning of data
 * transmission.  The means that until we send the first few ACK's the
 * sender will sit on his end and only queue most of his data, because
 * he can only send snd_cwnd unacked packets at any given time.  For
 * each ACK we send, he increments snd_cwnd and transmits more of his
 * queue.  -DaveM
 */
627
static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
628
{
629
	struct tcp_sock *tp = tcp_sk(sk);
630
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
631 632
	u32 now;

633
	inet_csk_schedule_ack(sk);
L
Linus Torvalds 已提交
634

635
	tcp_measure_rcv_mss(sk, skb);
L
Linus Torvalds 已提交
636 637

	tcp_rcv_rtt_measure(tp);
638

L
Linus Torvalds 已提交
639 640
	now = tcp_time_stamp;

641
	if (!icsk->icsk_ack.ato) {
L
Linus Torvalds 已提交
642 643 644
		/* The _first_ data packet received, initialize
		 * delayed ACK engine.
		 */
645 646
		tcp_incr_quickack(sk);
		icsk->icsk_ack.ato = TCP_ATO_MIN;
L
Linus Torvalds 已提交
647
	} else {
648
		int m = now - icsk->icsk_ack.lrcvtime;
L
Linus Torvalds 已提交
649

650
		if (m <= TCP_ATO_MIN / 2) {
L
Linus Torvalds 已提交
651
			/* The fastest case is the first. */
652 653 654 655 656 657
			icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2;
		} else if (m < icsk->icsk_ack.ato) {
			icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m;
			if (icsk->icsk_ack.ato > icsk->icsk_rto)
				icsk->icsk_ack.ato = icsk->icsk_rto;
		} else if (m > icsk->icsk_rto) {
S
Stephen Hemminger 已提交
658
			/* Too long gap. Apparently sender failed to
L
Linus Torvalds 已提交
659 660
			 * restart window, so that we send ACKs quickly.
			 */
661
			tcp_incr_quickack(sk);
662
			sk_mem_reclaim(sk);
L
Linus Torvalds 已提交
663 664
		}
	}
665
	icsk->icsk_ack.lrcvtime = now;
L
Linus Torvalds 已提交
666

667
	tcp_ecn_check_ce(tp, skb);
L
Linus Torvalds 已提交
668 669

	if (skb->len >= 128)
670
		tcp_grow_window(sk, skb);
L
Linus Torvalds 已提交
671 672 673 674 675 676 677 678 679 680 681
}

/* Called to compute a smoothed rtt estimate. The data fed to this
 * routine either comes from timestamps, or from segments that were
 * known _not_ to have been retransmitted [see Karn/Partridge
 * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
 * piece by Van Jacobson.
 * NOTE: the next three routines used to be one big routine.
 * To save cycles in the RFC 1323 implementation it was better to break
 * it up into three procedures. -- erics
 */
682
static void tcp_rtt_estimator(struct sock *sk, long mrtt_us)
L
Linus Torvalds 已提交
683
{
684
	struct tcp_sock *tp = tcp_sk(sk);
685 686
	long m = mrtt_us; /* RTT */
	u32 srtt = tp->srtt_us;
L
Linus Torvalds 已提交
687 688 689 690

	/*	The following amusing code comes from Jacobson's
	 *	article in SIGCOMM '88.  Note that rtt and mdev
	 *	are scaled versions of rtt and mean deviation.
691
	 *	This is designed to be as fast as possible
L
Linus Torvalds 已提交
692 693 694 695 696 697 698
	 *	m stands for "measurement".
	 *
	 *	On a 1990 paper the rto value is changed to:
	 *	RTO = rtt + 4 * mdev
	 *
	 * Funny. This algorithm seems to be very broken.
	 * These formulae increase RTO, when it should be decreased, increase
S
Stephen Hemminger 已提交
699
	 * too slowly, when it should be increased quickly, decrease too quickly
L
Linus Torvalds 已提交
700 701 702 703
	 * etc. I guess in BSD RTO takes ONE value, so that it is absolutely
	 * does not matter how to _calculate_ it. Seems, it was trap
	 * that VJ failed to avoid. 8)
	 */
704 705 706
	if (srtt != 0) {
		m -= (srtt >> 3);	/* m is now error in rtt est */
		srtt += m;		/* rtt = 7/8 rtt + 1/8 new */
L
Linus Torvalds 已提交
707 708
		if (m < 0) {
			m = -m;		/* m is now abs(error) */
709
			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
L
Linus Torvalds 已提交
710 711 712 713 714 715 716 717 718 719 720
			/* This is similar to one of Eifel findings.
			 * Eifel blocks mdev updates when rtt decreases.
			 * This solution is a bit different: we use finer gain
			 * for mdev in this case (alpha*beta).
			 * Like Eifel it also prevents growth of rto,
			 * but also it limits too fast rto decreases,
			 * happening in pure Eifel.
			 */
			if (m > 0)
				m >>= 3;
		} else {
721
			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
L
Linus Torvalds 已提交
722
		}
723 724 725 726 727
		tp->mdev_us += m;		/* mdev = 3/4 mdev + 1/4 new */
		if (tp->mdev_us > tp->mdev_max_us) {
			tp->mdev_max_us = tp->mdev_us;
			if (tp->mdev_max_us > tp->rttvar_us)
				tp->rttvar_us = tp->mdev_max_us;
L
Linus Torvalds 已提交
728 729
		}
		if (after(tp->snd_una, tp->rtt_seq)) {
730 731
			if (tp->mdev_max_us < tp->rttvar_us)
				tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2;
L
Linus Torvalds 已提交
732
			tp->rtt_seq = tp->snd_nxt;
733
			tp->mdev_max_us = tcp_rto_min_us(sk);
L
Linus Torvalds 已提交
734 735 736
		}
	} else {
		/* no previous measure. */
737
		srtt = m << 3;		/* take the measured time to be rtt */
738 739 740
		tp->mdev_us = m << 1;	/* make sure rto = 3*rtt */
		tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk));
		tp->mdev_max_us = tp->rttvar_us;
L
Linus Torvalds 已提交
741 742
		tp->rtt_seq = tp->snd_nxt;
	}
743
	tp->srtt_us = max(1U, srtt);
L
Linus Torvalds 已提交
744 745
}

E
Eric Dumazet 已提交
746 747 748 749 750 751 752 753 754 755 756 757
/* Set the sk_pacing_rate to allow proper sizing of TSO packets.
 * Note: TCP stack does not yet implement pacing.
 * FQ packet scheduler can be used to implement cheap but effective
 * TCP pacing, to smooth the burst on large writes when packets
 * in flight is significantly lower than cwnd (or rwin)
 */
static void tcp_update_pacing_rate(struct sock *sk)
{
	const struct tcp_sock *tp = tcp_sk(sk);
	u64 rate;

	/* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
758
	rate = (u64)tp->mss_cache * 2 * (USEC_PER_SEC << 3);
E
Eric Dumazet 已提交
759 760 761

	rate *= max(tp->snd_cwnd, tp->packets_out);

762 763
	if (likely(tp->srtt_us))
		do_div(rate, tp->srtt_us);
E
Eric Dumazet 已提交
764

765 766 767 768 769 770
	/* ACCESS_ONCE() is needed because sch_fq fetches sk_pacing_rate
	 * without any lock. We want to make sure compiler wont store
	 * intermediate values in this location.
	 */
	ACCESS_ONCE(sk->sk_pacing_rate) = min_t(u64, rate,
						sk->sk_max_pacing_rate);
E
Eric Dumazet 已提交
771 772
}

L
Linus Torvalds 已提交
773 774 775
/* Calculate rto without backoff.  This is the second half of Van Jacobson's
 * routine referred to above.
 */
776
static void tcp_set_rto(struct sock *sk)
L
Linus Torvalds 已提交
777
{
778
	const struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
779 780 781 782 783 784 785 786
	/* Old crap is replaced with new one. 8)
	 *
	 * More seriously:
	 * 1. If rtt variance happened to be less 50msec, it is hallucination.
	 *    It cannot be less due to utterly erratic ACK generation made
	 *    at least by solaris and freebsd. "Erratic ACKs" has _nothing_
	 *    to do with delayed acks, because at cwnd>2 true delack timeout
	 *    is invisible. Actually, Linux-2.4 also generates erratic
S
Stephen Hemminger 已提交
787
	 *    ACKs in some circumstances.
L
Linus Torvalds 已提交
788
	 */
789
	inet_csk(sk)->icsk_rto = __tcp_set_rto(tp);
L
Linus Torvalds 已提交
790 791 792 793

	/* 2. Fixups made earlier cannot be right.
	 *    If we do not estimate RTO correctly without them,
	 *    all the algo is pure shit and should be replaced
S
Stephen Hemminger 已提交
794
	 *    with correct one. It is exactly, which we pretend to do.
L
Linus Torvalds 已提交
795 796
	 */

797 798 799
	/* NOTE: clamping at TCP_RTO_MIN is not required, current algo
	 * guarantees that rto is higher.
	 */
800
	tcp_bound_rto(sk);
L
Linus Torvalds 已提交
801 802
}

803
__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst)
L
Linus Torvalds 已提交
804 805 806
{
	__u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0);

807
	if (!cwnd)
808
		cwnd = TCP_INIT_CWND;
L
Linus Torvalds 已提交
809 810 811
	return min_t(__u32, cwnd, tp->snd_cwnd_clamp);
}

812 813 814 815
/*
 * Packet counting of FACK is based on in-order assumptions, therefore TCP
 * disables it when reordering is detected
 */
816
void tcp_disable_fack(struct tcp_sock *tp)
817
{
818 819 820
	/* RFC3517 uses different metric in lost marker => reset on change */
	if (tcp_is_fack(tp))
		tp->lost_skb_hint = NULL;
821
	tp->rx_opt.sack_ok &= ~TCP_FACK_ENABLED;
822 823
}

824
/* Take a notice that peer is sending D-SACKs */
825 826
static void tcp_dsack_seen(struct tcp_sock *tp)
{
827
	tp->rx_opt.sack_ok |= TCP_DSACK_SEEN;
828 829
}

830 831
static void tcp_update_reordering(struct sock *sk, const int metric,
				  const int ts)
L
Linus Torvalds 已提交
832
{
833
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
834
	if (metric > tp->reordering) {
835 836
		int mib_idx;

837
		tp->reordering = min(sysctl_tcp_max_reordering, metric);
L
Linus Torvalds 已提交
838 839 840

		/* This exciting event is worth to be remembered. 8) */
		if (ts)
841
			mib_idx = LINUX_MIB_TCPTSREORDER;
842
		else if (tcp_is_reno(tp))
843
			mib_idx = LINUX_MIB_TCPRENOREORDER;
844
		else if (tcp_is_fack(tp))
845
			mib_idx = LINUX_MIB_TCPFACKREORDER;
L
Linus Torvalds 已提交
846
		else
847 848
			mib_idx = LINUX_MIB_TCPSACKREORDER;

849
		NET_INC_STATS_BH(sock_net(sk), mib_idx);
L
Linus Torvalds 已提交
850
#if FASTRETRANS_DEBUG > 1
851 852 853 854 855 856
		pr_debug("Disorder%d %d %u f%u s%u rr%d\n",
			 tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state,
			 tp->reordering,
			 tp->fackets_out,
			 tp->sacked_out,
			 tp->undo_marker ? tp->undo_retrans : 0);
L
Linus Torvalds 已提交
857
#endif
858
		tcp_disable_fack(tp);
L
Linus Torvalds 已提交
859
	}
Y
Yuchung Cheng 已提交
860 861 862

	if (metric > 0)
		tcp_disable_early_retrans(tp);
L
Linus Torvalds 已提交
863 864
}

865
/* This must be called before lost_out is incremented */
866 867
static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)
{
868
	if ((tp->retransmit_skb_hint == NULL) ||
869 870
	    before(TCP_SKB_CB(skb)->seq,
		   TCP_SKB_CB(tp->retransmit_skb_hint)->seq))
871 872 873 874 875
		tp->retransmit_skb_hint = skb;

	if (!tp->lost_out ||
	    after(TCP_SKB_CB(skb)->end_seq, tp->retransmit_high))
		tp->retransmit_high = TCP_SKB_CB(skb)->end_seq;
876 877
}

878 879 880 881 882 883 884 885 886 887
static void tcp_skb_mark_lost(struct tcp_sock *tp, struct sk_buff *skb)
{
	if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) {
		tcp_verify_retransmit_hint(tp, skb);

		tp->lost_out += tcp_skb_pcount(skb);
		TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
	}
}

888 889
static void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp,
					    struct sk_buff *skb)
890 891 892 893 894 895 896 897 898
{
	tcp_verify_retransmit_hint(tp, skb);

	if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) {
		tp->lost_out += tcp_skb_pcount(skb);
		TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
	}
}

L
Linus Torvalds 已提交
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
/* This procedure tags the retransmission queue when SACKs arrive.
 *
 * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L).
 * Packets in queue with these bits set are counted in variables
 * sacked_out, retrans_out and lost_out, correspondingly.
 *
 * Valid combinations are:
 * Tag  InFlight	Description
 * 0	1		- orig segment is in flight.
 * S	0		- nothing flies, orig reached receiver.
 * L	0		- nothing flies, orig lost by net.
 * R	2		- both orig and retransmit are in flight.
 * L|R	1		- orig is lost, retransmit is in flight.
 * S|R  1		- orig reached receiver, retrans is still in flight.
 * (L|S|R is logically valid, it could occur when L|R is sacked,
 *  but it is equivalent to plain S and code short-curcuits it to S.
 *  L|S is logically invalid, it would mean -1 packet in flight 8))
 *
 * These 6 states form finite state machine, controlled by the following events:
 * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue())
 * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue())
920
 * 3. Loss detection event of two flavors:
L
Linus Torvalds 已提交
921 922
 *	A. Scoreboard estimator decided the packet is lost.
 *	   A'. Reno "three dupacks" marks head of queue lost.
923 924
 *	   A''. Its FACK modification, head until snd.fack is lost.
 *	B. SACK arrives sacking SND.NXT at the moment, when the
L
Linus Torvalds 已提交
925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943
 *	   segment was retransmitted.
 * 4. D-SACK added new rule: D-SACK changes any tag to S.
 *
 * It is pleasant to note, that state diagram turns out to be commutative,
 * so that we are allowed not to be bothered by order of our actions,
 * when multiple events arrive simultaneously. (see the function below).
 *
 * Reordering detection.
 * --------------------
 * Reordering metric is maximal distance, which a packet can be displaced
 * in packet stream. With SACKs we can estimate it:
 *
 * 1. SACK fills old hole and the corresponding segment was not
 *    ever retransmitted -> reordering. Alas, we cannot use it
 *    when segment was retransmitted.
 * 2. The last flaw is solved with D-SACK. D-SACK arrives
 *    for retransmitted and already SACKed segment -> reordering..
 * Both of these heuristics are not used in Loss state, when we cannot
 * account for retransmits accurately.
I
Ilpo Järvinen 已提交
944 945 946 947 948 949 950
 *
 * SACK block validation.
 * ----------------------
 *
 * SACK block range validation checks that the received SACK block fits to
 * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT.
 * Note that SND.UNA is not included to the range though being valid because
951 952 953 954 955 956 957 958 959
 * it means that the receiver is rather inconsistent with itself reporting
 * SACK reneging when it should advance SND.UNA. Such SACK block this is
 * perfectly valid, however, in light of RFC2018 which explicitly states
 * that "SACK block MUST reflect the newest segment.  Even if the newest
 * segment is going to be discarded ...", not that it looks very clever
 * in case of head skb. Due to potentional receiver driven attacks, we
 * choose to avoid immediate execution of a walk in write queue due to
 * reneging and defer head skb's loss recovery to standard loss recovery
 * procedure that will eventually trigger (nothing forbids us doing this).
I
Ilpo Järvinen 已提交
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981
 *
 * Implements also blockage to start_seq wrap-around. Problem lies in the
 * fact that though start_seq (s) is before end_seq (i.e., not reversed),
 * there's no guarantee that it will be before snd_nxt (n). The problem
 * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt
 * wrap (s_w):
 *
 *         <- outs wnd ->                          <- wrapzone ->
 *         u     e      n                         u_w   e_w  s n_w
 *         |     |      |                          |     |   |  |
 * |<------------+------+----- TCP seqno space --------------+---------->|
 * ...-- <2^31 ->|                                           |<--------...
 * ...---- >2^31 ------>|                                    |<--------...
 *
 * Current code wouldn't be vulnerable but it's better still to discard such
 * crazy SACK blocks. Doing this check for start_seq alone closes somewhat
 * similar case (end_seq after snd_nxt wrap) as earlier reversed check in
 * snd_nxt wrap -> snd_una region will then become "well defined", i.e.,
 * equal to the ideal case (infinite seqno space without wrap caused issues).
 *
 * With D-SACK the lower bound is extended to cover sequence space below
 * SND.UNA down to undo_marker, which is the last point of interest. Yet
982
 * again, D-SACK block must not to go across snd_una (for the same reason as
I
Ilpo Järvinen 已提交
983 984 985 986 987 988 989 990 991
 * for the normal SACK blocks, explained above). But there all simplicity
 * ends, TCP might receive valid D-SACKs below that. As long as they reside
 * fully below undo_marker they do not affect behavior in anyway and can
 * therefore be safely ignored. In rare cases (which are more or less
 * theoretical ones), the D-SACK will nicely cross that boundary due to skb
 * fragmentation and packet reordering past skb's retransmission. To consider
 * them correctly, the acceptable range must be extended even more though
 * the exact amount is rather hard to quantify. However, tp->max_window can
 * be used as an exaggerated estimate.
L
Linus Torvalds 已提交
992
 */
E
Eric Dumazet 已提交
993 994
static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack,
				   u32 start_seq, u32 end_seq)
I
Ilpo Järvinen 已提交
995 996 997
{
	/* Too far in future, or reversed (interpretation is ambiguous) */
	if (after(end_seq, tp->snd_nxt) || !before(start_seq, end_seq))
E
Eric Dumazet 已提交
998
		return false;
I
Ilpo Järvinen 已提交
999 1000 1001

	/* Nasty start_seq wrap-around check (see comments above) */
	if (!before(start_seq, tp->snd_nxt))
E
Eric Dumazet 已提交
1002
		return false;
I
Ilpo Järvinen 已提交
1003

1004
	/* In outstanding window? ...This is valid exit for D-SACKs too.
I
Ilpo Järvinen 已提交
1005 1006 1007
	 * start_seq == snd_una is non-sensical (see comments above)
	 */
	if (after(start_seq, tp->snd_una))
E
Eric Dumazet 已提交
1008
		return true;
I
Ilpo Järvinen 已提交
1009 1010

	if (!is_dsack || !tp->undo_marker)
E
Eric Dumazet 已提交
1011
		return false;
I
Ilpo Järvinen 已提交
1012 1013

	/* ...Then it's D-SACK, and must reside below snd_una completely */
Z
Zheng Yan 已提交
1014
	if (after(end_seq, tp->snd_una))
E
Eric Dumazet 已提交
1015
		return false;
I
Ilpo Järvinen 已提交
1016 1017

	if (!before(start_seq, tp->undo_marker))
E
Eric Dumazet 已提交
1018
		return true;
I
Ilpo Järvinen 已提交
1019 1020 1021

	/* Too old */
	if (!after(end_seq, tp->undo_marker))
E
Eric Dumazet 已提交
1022
		return false;
I
Ilpo Järvinen 已提交
1023 1024 1025 1026 1027 1028 1029

	/* Undo_marker boundary crossing (overestimates a lot). Known already:
	 *   start_seq < undo_marker and end_seq >= undo_marker.
	 */
	return !before(start_seq, end_seq - tp->max_window);
}

1030
/* Check for lost retransmit. This superb idea is borrowed from "ratehalving".
1031
 * Event "B". Later note: FACK people cheated me again 8), we have to account
1032
 * for reordering! Ugly, but should help.
1033 1034 1035
 *
 * Search retransmitted skbs from write_queue that were sent when snd_nxt was
 * less than what is now known to be received by the other end (derived from
1036 1037
 * highest SACK block). Also calculate the lowest snd_nxt among the remaining
 * retransmitted skbs to avoid some costly processing per ACKs.
1038
 */
1039
static void tcp_mark_lost_retrans(struct sock *sk)
1040
{
1041
	const struct inet_connection_sock *icsk = inet_csk(sk);
1042 1043
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb;
1044
	int cnt = 0;
1045
	u32 new_low_seq = tp->snd_nxt;
1046
	u32 received_upto = tcp_highest_sack_seq(tp);
1047 1048 1049 1050

	if (!tcp_is_fack(tp) || !tp->retrans_out ||
	    !after(received_upto, tp->lost_retrans_low) ||
	    icsk->icsk_ca_state != TCP_CA_Recovery)
1051
		return;
1052 1053 1054 1055 1056 1057

	tcp_for_write_queue(skb, sk) {
		u32 ack_seq = TCP_SKB_CB(skb)->ack_seq;

		if (skb == tcp_send_head(sk))
			break;
1058
		if (cnt == tp->retrans_out)
1059 1060 1061 1062
			break;
		if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
			continue;

1063 1064 1065
		if (!(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS))
			continue;

1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
		/* TODO: We would like to get rid of tcp_is_fack(tp) only
		 * constraint here (see above) but figuring out that at
		 * least tp->reordering SACK blocks reside between ack_seq
		 * and received_upto is not easy task to do cheaply with
		 * the available datastructures.
		 *
		 * Whether FACK should check here for tp->reordering segs
		 * in-between one could argue for either way (it would be
		 * rather simple to implement as we could count fack_count
		 * during the walk and do tp->fackets_out - fack_count).
		 */
		if (after(received_upto, ack_seq)) {
1078 1079 1080
			TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
			tp->retrans_out -= tcp_skb_pcount(skb);

1081
			tcp_skb_mark_lost_uncond_verify(tp, skb);
1082
			NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPLOSTRETRANSMIT);
1083
		} else {
1084
			if (before(ack_seq, new_low_seq))
1085
				new_low_seq = ack_seq;
1086
			cnt += tcp_skb_pcount(skb);
1087 1088
		}
	}
1089 1090 1091

	if (tp->retrans_out)
		tp->lost_retrans_low = new_low_seq;
1092
}
I
Ilpo Järvinen 已提交
1093

E
Eric Dumazet 已提交
1094 1095 1096
static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb,
			    struct tcp_sack_block_wire *sp, int num_sacks,
			    u32 prior_snd_una)
1097
{
1098
	struct tcp_sock *tp = tcp_sk(sk);
1099 1100
	u32 start_seq_0 = get_unaligned_be32(&sp[0].start_seq);
	u32 end_seq_0 = get_unaligned_be32(&sp[0].end_seq);
E
Eric Dumazet 已提交
1101
	bool dup_sack = false;
1102 1103

	if (before(start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) {
E
Eric Dumazet 已提交
1104
		dup_sack = true;
1105
		tcp_dsack_seen(tp);
1106
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPDSACKRECV);
1107
	} else if (num_sacks > 1) {
1108 1109
		u32 end_seq_1 = get_unaligned_be32(&sp[1].end_seq);
		u32 start_seq_1 = get_unaligned_be32(&sp[1].start_seq);
1110 1111 1112

		if (!after(end_seq_0, end_seq_1) &&
		    !before(start_seq_0, start_seq_1)) {
E
Eric Dumazet 已提交
1113
			dup_sack = true;
1114
			tcp_dsack_seen(tp);
1115 1116
			NET_INC_STATS_BH(sock_net(sk),
					LINUX_MIB_TCPDSACKOFORECV);
1117 1118 1119 1120
		}
	}

	/* D-SACK for already forgotten data... Do dumb counting. */
Y
Yuchung Cheng 已提交
1121
	if (dup_sack && tp->undo_marker && tp->undo_retrans > 0 &&
1122 1123 1124 1125 1126 1127 1128
	    !after(end_seq_0, prior_snd_una) &&
	    after(end_seq_0, tp->undo_marker))
		tp->undo_retrans--;

	return dup_sack;
}

1129
struct tcp_sacktag_state {
1130 1131 1132 1133
	int	reord;
	int	fack_count;
	long	rtt_us; /* RTT measured by SACKing never-retransmitted data */
	int	flag;
1134 1135
};

1136 1137 1138 1139 1140
/* Check if skb is fully within the SACK block. In presence of GSO skbs,
 * the incoming SACK may not exactly match but we can find smaller MSS
 * aligned portion of it that matches. Therefore we might need to fragment
 * which may fail and creates some hassle (caller must handle error case
 * returns).
1141 1142
 *
 * FIXME: this could be merged to shift decision code
1143
 */
1144
static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb,
E
Eric Dumazet 已提交
1145
				  u32 start_seq, u32 end_seq)
1146
{
E
Eric Dumazet 已提交
1147 1148
	int err;
	bool in_sack;
1149
	unsigned int pkt_len;
1150
	unsigned int mss;
1151 1152 1153 1154 1155 1156

	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);

	if (tcp_skb_pcount(skb) > 1 && !in_sack &&
	    after(TCP_SKB_CB(skb)->end_seq, start_seq)) {
1157
		mss = tcp_skb_mss(skb);
1158 1159
		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);

1160
		if (!in_sack) {
1161
			pkt_len = start_seq - TCP_SKB_CB(skb)->seq;
1162 1163 1164
			if (pkt_len < mss)
				pkt_len = mss;
		} else {
1165
			pkt_len = end_seq - TCP_SKB_CB(skb)->seq;
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
			if (pkt_len < mss)
				return -EINVAL;
		}

		/* Round if necessary so that SACKs cover only full MSSes
		 * and/or the remaining small portion (if present)
		 */
		if (pkt_len > mss) {
			unsigned int new_len = (pkt_len / mss) * mss;
			if (!in_sack && new_len < pkt_len) {
				new_len += mss;
1177
				if (new_len >= skb->len)
1178 1179 1180 1181
					return 0;
			}
			pkt_len = new_len;
		}
1182
		err = tcp_fragment(sk, skb, pkt_len, mss, GFP_ATOMIC);
1183 1184 1185 1186 1187 1188 1189
		if (err < 0)
			return err;
	}

	return in_sack;
}

1190 1191 1192 1193
/* Mark the given newly-SACKed range as such, adjusting counters and hints. */
static u8 tcp_sacktag_one(struct sock *sk,
			  struct tcp_sacktag_state *state, u8 sacked,
			  u32 start_seq, u32 end_seq,
1194 1195
			  int dup_sack, int pcount,
			  const struct skb_mstamp *xmit_time)
1196
{
1197
	struct tcp_sock *tp = tcp_sk(sk);
1198
	int fack_count = state->fack_count;
1199 1200 1201

	/* Account D-SACK for retransmitted packet. */
	if (dup_sack && (sacked & TCPCB_RETRANS)) {
Y
Yuchung Cheng 已提交
1202
		if (tp->undo_marker && tp->undo_retrans > 0 &&
1203
		    after(end_seq, tp->undo_marker))
1204
			tp->undo_retrans--;
1205
		if (sacked & TCPCB_SACKED_ACKED)
1206
			state->reord = min(fack_count, state->reord);
1207 1208 1209
	}

	/* Nothing to do; acked frame is about to be dropped (was ACKed). */
1210
	if (!after(end_seq, tp->snd_una))
1211
		return sacked;
1212 1213 1214 1215 1216 1217 1218 1219

	if (!(sacked & TCPCB_SACKED_ACKED)) {
		if (sacked & TCPCB_SACKED_RETRANS) {
			/* If the segment is not tagged as lost,
			 * we do not clear RETRANS, believing
			 * that retransmission is still in flight.
			 */
			if (sacked & TCPCB_LOST) {
1220
				sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS);
1221 1222
				tp->lost_out -= pcount;
				tp->retrans_out -= pcount;
1223 1224 1225 1226 1227 1228
			}
		} else {
			if (!(sacked & TCPCB_RETRANS)) {
				/* New sack for not retransmitted frame,
				 * which was in hole. It is reordering.
				 */
1229
				if (before(start_seq,
1230
					   tcp_highest_sack_seq(tp)))
1231 1232
					state->reord = min(fack_count,
							   state->reord);
Y
Yuchung Cheng 已提交
1233 1234
				if (!after(end_seq, tp->high_seq))
					state->flag |= FLAG_ORIG_SACK_ACKED;
Y
Yuchung Cheng 已提交
1235
				/* Pick the earliest sequence sacked for RTT */
1236 1237 1238 1239 1240 1241 1242
				if (state->rtt_us < 0) {
					struct skb_mstamp now;

					skb_mstamp_get(&now);
					state->rtt_us = skb_mstamp_us_delta(&now,
								xmit_time);
				}
1243 1244 1245
			}

			if (sacked & TCPCB_LOST) {
1246
				sacked &= ~TCPCB_LOST;
1247
				tp->lost_out -= pcount;
1248 1249 1250
			}
		}

1251 1252
		sacked |= TCPCB_SACKED_ACKED;
		state->flag |= FLAG_DATA_SACKED;
1253
		tp->sacked_out += pcount;
1254

1255
		fack_count += pcount;
1256 1257 1258

		/* Lost marker hint past SACKed? Tweak RFC3517 cnt */
		if (!tcp_is_fack(tp) && (tp->lost_skb_hint != NULL) &&
1259
		    before(start_seq, TCP_SKB_CB(tp->lost_skb_hint)->seq))
1260
			tp->lost_cnt_hint += pcount;
1261 1262 1263 1264 1265 1266 1267 1268 1269

		if (fack_count > tp->fackets_out)
			tp->fackets_out = fack_count;
	}

	/* D-SACK. We can detect redundant retransmission in S|R and plain R
	 * frames and clear it. undo_retrans is decreased above, L|R frames
	 * are accounted above as well.
	 */
1270 1271
	if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) {
		sacked &= ~TCPCB_SACKED_RETRANS;
1272
		tp->retrans_out -= pcount;
1273 1274
	}

1275
	return sacked;
1276 1277
}

1278 1279 1280
/* Shift newly-SACKed bytes from this skb to the immediately previous
 * already-SACKed sk_buff. Mark the newly-SACKed bytes as such.
 */
E
Eric Dumazet 已提交
1281 1282 1283 1284
static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *skb,
			    struct tcp_sacktag_state *state,
			    unsigned int pcount, int shifted, int mss,
			    bool dup_sack)
1285 1286
{
	struct tcp_sock *tp = tcp_sk(sk);
1287
	struct sk_buff *prev = tcp_write_queue_prev(sk, skb);
1288 1289
	u32 start_seq = TCP_SKB_CB(skb)->seq;	/* start of newly-SACKed */
	u32 end_seq = start_seq + shifted;	/* end of newly-SACKed */
1290 1291 1292

	BUG_ON(!pcount);

1293 1294 1295 1296 1297 1298 1299
	/* Adjust counters and hints for the newly sacked sequence
	 * range but discard the return value since prev is already
	 * marked. We must tag the range first because the seq
	 * advancement below implicitly advances
	 * tcp_highest_sack_seq() when skb is highest_sack.
	 */
	tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked,
Y
Yuchung Cheng 已提交
1300
			start_seq, end_seq, dup_sack, pcount,
1301
			&skb->skb_mstamp);
1302 1303

	if (skb == tp->lost_skb_hint)
1304 1305
		tp->lost_cnt_hint += pcount;

1306 1307 1308
	TCP_SKB_CB(prev)->end_seq += shifted;
	TCP_SKB_CB(skb)->seq += shifted;

1309 1310 1311
	tcp_skb_pcount_add(prev, pcount);
	BUG_ON(tcp_skb_pcount(skb) < pcount);
	tcp_skb_pcount_add(skb, -pcount);
1312 1313 1314 1315 1316 1317 1318 1319

	/* When we're adding to gso_segs == 1, gso_size will be zero,
	 * in theory this shouldn't be necessary but as long as DSACK
	 * code can come after this skb later on it's better to keep
	 * setting gso_size to something.
	 */
	if (!skb_shinfo(prev)->gso_size) {
		skb_shinfo(prev)->gso_size = mss;
1320
		skb_shinfo(prev)->gso_type = sk->sk_gso_type;
1321 1322 1323
	}

	/* CHECKME: To clear or not to clear? Mimics normal skb currently */
1324
	if (tcp_skb_pcount(skb) <= 1) {
1325
		skb_shinfo(skb)->gso_size = 0;
1326
		skb_shinfo(skb)->gso_type = 0;
1327 1328 1329 1330 1331 1332 1333
	}

	/* Difference in this won't matter, both ACKed by the same cumul. ACK */
	TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS);

	if (skb->len > 0) {
		BUG_ON(!tcp_skb_pcount(skb));
1334
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKSHIFTED);
E
Eric Dumazet 已提交
1335
		return false;
1336 1337 1338 1339
	}

	/* Whole SKB was eaten :-) */

1340 1341 1342 1343 1344 1345 1346
	if (skb == tp->retransmit_skb_hint)
		tp->retransmit_skb_hint = prev;
	if (skb == tp->lost_skb_hint) {
		tp->lost_skb_hint = prev;
		tp->lost_cnt_hint -= tcp_skb_pcount(prev);
	}

1347 1348 1349 1350
	TCP_SKB_CB(prev)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
		TCP_SKB_CB(prev)->end_seq++;

1351 1352 1353 1354 1355 1356
	if (skb == tcp_highest_sack(sk))
		tcp_advance_highest_sack(sk, skb);

	tcp_unlink_write_queue(skb, sk);
	sk_wmem_free_skb(sk, skb);

1357 1358
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKMERGED);

E
Eric Dumazet 已提交
1359
	return true;
1360 1361 1362 1363 1364
}

/* I wish gso_size would have a bit more sane initialization than
 * something-or-zero which complicates things
 */
1365
static int tcp_skb_seglen(const struct sk_buff *skb)
1366
{
1367
	return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb);
1368 1369 1370
}

/* Shifting pages past head area doesn't work */
1371
static int skb_can_shift(const struct sk_buff *skb)
1372 1373 1374 1375 1376 1377 1378 1379
{
	return !skb_headlen(skb) && skb_is_nonlinear(skb);
}

/* Try collapsing SACK blocks spanning across multiple skbs to a single
 * skb.
 */
static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb,
1380
					  struct tcp_sacktag_state *state,
1381
					  u32 start_seq, u32 end_seq,
E
Eric Dumazet 已提交
1382
					  bool dup_sack)
1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *prev;
	int mss;
	int pcount = 0;
	int len;
	int in_sack;

	if (!sk_can_gso(sk))
		goto fallback;

	/* Normally R but no L won't result in plain S */
	if (!dup_sack &&
1396
	    (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS)
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
		goto fallback;
	if (!skb_can_shift(skb))
		goto fallback;
	/* This frame is about to be dropped (was ACKed). */
	if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
		goto fallback;

	/* Can only happen with delayed DSACK + discard craziness */
	if (unlikely(skb == tcp_write_queue_head(sk)))
		goto fallback;
	prev = tcp_write_queue_prev(sk, skb);

	if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED)
		goto fallback;

	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);

	if (in_sack) {
		len = skb->len;
		pcount = tcp_skb_pcount(skb);
1418
		mss = tcp_skb_seglen(skb);
1419 1420 1421 1422

		/* TODO: Fix DSACKs to not fragment already SACKed and we can
		 * drop this restriction as unnecessary
		 */
1423
		if (mss != tcp_skb_seglen(prev))
1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463
			goto fallback;
	} else {
		if (!after(TCP_SKB_CB(skb)->end_seq, start_seq))
			goto noop;
		/* CHECKME: This is non-MSS split case only?, this will
		 * cause skipped skbs due to advancing loop btw, original
		 * has that feature too
		 */
		if (tcp_skb_pcount(skb) <= 1)
			goto noop;

		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
		if (!in_sack) {
			/* TODO: head merge to next could be attempted here
			 * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)),
			 * though it might not be worth of the additional hassle
			 *
			 * ...we can probably just fallback to what was done
			 * previously. We could try merging non-SACKed ones
			 * as well but it probably isn't going to buy off
			 * because later SACKs might again split them, and
			 * it would make skb timestamp tracking considerably
			 * harder problem.
			 */
			goto fallback;
		}

		len = end_seq - TCP_SKB_CB(skb)->seq;
		BUG_ON(len < 0);
		BUG_ON(len > skb->len);

		/* MSS boundaries should be honoured or else pcount will
		 * severely break even though it makes things bit trickier.
		 * Optimize common case to avoid most of the divides
		 */
		mss = tcp_skb_mss(skb);

		/* TODO: Fix DSACKs to not fragment already SACKed and we can
		 * drop this restriction as unnecessary
		 */
1464
		if (mss != tcp_skb_seglen(prev))
1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
			goto fallback;

		if (len == mss) {
			pcount = 1;
		} else if (len < mss) {
			goto noop;
		} else {
			pcount = len / mss;
			len = pcount * mss;
		}
	}

1477 1478 1479 1480
	/* tcp_sacktag_one() won't SACK-tag ranges below snd_una */
	if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una))
		goto fallback;

1481 1482
	if (!skb_shift(prev, skb, len))
		goto fallback;
I
Ilpo Järvinen 已提交
1483
	if (!tcp_shifted_skb(sk, skb, state, pcount, len, mss, dup_sack))
1484 1485 1486 1487 1488 1489 1490 1491 1492
		goto out;

	/* Hole filled allows collapsing with the next as well, this is very
	 * useful when hole on every nth skb pattern happens
	 */
	if (prev == tcp_write_queue_tail(sk))
		goto out;
	skb = tcp_write_queue_next(sk, prev);

1493 1494 1495
	if (!skb_can_shift(skb) ||
	    (skb == tcp_send_head(sk)) ||
	    ((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) ||
1496
	    (mss != tcp_skb_seglen(skb)))
1497 1498 1499 1500 1501
		goto out;

	len = skb->len;
	if (skb_shift(prev, skb, len)) {
		pcount += tcp_skb_pcount(skb);
I
Ilpo Järvinen 已提交
1502
		tcp_shifted_skb(sk, skb, state, tcp_skb_pcount(skb), len, mss, 0);
1503 1504 1505
	}

out:
1506
	state->fack_count += pcount;
1507 1508 1509 1510 1511 1512
	return prev;

noop:
	return skb;

fallback:
1513
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK);
1514 1515 1516
	return NULL;
}

1517 1518
static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk,
					struct tcp_sack_block *next_dup,
1519
					struct tcp_sacktag_state *state,
1520
					u32 start_seq, u32 end_seq,
E
Eric Dumazet 已提交
1521
					bool dup_sack_in)
1522
{
1523 1524 1525
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *tmp;

1526 1527
	tcp_for_write_queue_from(skb, sk) {
		int in_sack = 0;
E
Eric Dumazet 已提交
1528
		bool dup_sack = dup_sack_in;
1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542

		if (skb == tcp_send_head(sk))
			break;

		/* queue is in-order => we can short-circuit the walk early */
		if (!before(TCP_SKB_CB(skb)->seq, end_seq))
			break;

		if ((next_dup != NULL) &&
		    before(TCP_SKB_CB(skb)->seq, next_dup->end_seq)) {
			in_sack = tcp_match_skb_to_sack(sk, skb,
							next_dup->start_seq,
							next_dup->end_seq);
			if (in_sack > 0)
E
Eric Dumazet 已提交
1543
				dup_sack = true;
1544 1545
		}

1546 1547 1548 1549 1550
		/* skb reference here is a bit tricky to get right, since
		 * shifting can eat and free both this skb and the next,
		 * so not even _safe variant of the loop is enough.
		 */
		if (in_sack <= 0) {
1551 1552
			tmp = tcp_shift_skb_data(sk, skb, state,
						 start_seq, end_seq, dup_sack);
1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
			if (tmp != NULL) {
				if (tmp != skb) {
					skb = tmp;
					continue;
				}

				in_sack = 0;
			} else {
				in_sack = tcp_match_skb_to_sack(sk, skb,
								start_seq,
								end_seq);
			}
		}

1567 1568 1569
		if (unlikely(in_sack < 0))
			break;

1570
		if (in_sack) {
1571 1572 1573 1574 1575 1576 1577
			TCP_SKB_CB(skb)->sacked =
				tcp_sacktag_one(sk,
						state,
						TCP_SKB_CB(skb)->sacked,
						TCP_SKB_CB(skb)->seq,
						TCP_SKB_CB(skb)->end_seq,
						dup_sack,
Y
Yuchung Cheng 已提交
1578
						tcp_skb_pcount(skb),
1579
						&skb->skb_mstamp);
1580

1581 1582 1583 1584 1585
			if (!before(TCP_SKB_CB(skb)->seq,
				    tcp_highest_sack_seq(tp)))
				tcp_advance_highest_sack(sk, skb);
		}

1586
		state->fack_count += tcp_skb_pcount(skb);
1587 1588 1589 1590 1591 1592 1593 1594
	}
	return skb;
}

/* Avoid all extra work that is being done by sacktag while walking in
 * a normal way
 */
static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk,
1595 1596
					struct tcp_sacktag_state *state,
					u32 skip_to_seq)
1597 1598 1599 1600 1601
{
	tcp_for_write_queue_from(skb, sk) {
		if (skb == tcp_send_head(sk))
			break;

I
Ilpo Järvinen 已提交
1602
		if (after(TCP_SKB_CB(skb)->end_seq, skip_to_seq))
1603
			break;
1604

1605
		state->fack_count += tcp_skb_pcount(skb);
1606 1607 1608 1609 1610 1611 1612
	}
	return skb;
}

static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb,
						struct sock *sk,
						struct tcp_sack_block *next_dup,
1613 1614
						struct tcp_sacktag_state *state,
						u32 skip_to_seq)
1615 1616 1617 1618 1619
{
	if (next_dup == NULL)
		return skb;

	if (before(next_dup->start_seq, skip_to_seq)) {
1620 1621 1622 1623
		skb = tcp_sacktag_skip(skb, sk, state, next_dup->start_seq);
		skb = tcp_sacktag_walk(skb, sk, NULL, state,
				       next_dup->start_seq, next_dup->end_seq,
				       1);
1624 1625 1626 1627 1628
	}

	return skb;
}

1629
static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache)
1630 1631 1632 1633
{
	return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
}

L
Linus Torvalds 已提交
1634
static int
1635
tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb,
1636
			u32 prior_snd_una, long *sack_rtt_us)
L
Linus Torvalds 已提交
1637 1638
{
	struct tcp_sock *tp = tcp_sk(sk);
1639 1640
	const unsigned char *ptr = (skb_transport_header(ack_skb) +
				    TCP_SKB_CB(ack_skb)->sacked);
1641
	struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2);
1642
	struct tcp_sack_block sp[TCP_NUM_SACKS];
1643
	struct tcp_sack_block *cache;
1644
	struct tcp_sacktag_state state;
1645
	struct sk_buff *skb;
1646
	int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3);
1647
	int used_sacks;
E
Eric Dumazet 已提交
1648
	bool found_dup_sack = false;
1649
	int i, j;
1650
	int first_sack_index;
L
Linus Torvalds 已提交
1651

1652 1653
	state.flag = 0;
	state.reord = tp->packets_out;
1654
	state.rtt_us = -1L;
1655

1656
	if (!tp->sacked_out) {
1657 1658
		if (WARN_ON(tp->fackets_out))
			tp->fackets_out = 0;
1659
		tcp_highest_sack_reset(sk);
1660
	}
L
Linus Torvalds 已提交
1661

1662
	found_dup_sack = tcp_check_dsack(sk, ack_skb, sp_wire,
1663 1664
					 num_sacks, prior_snd_una);
	if (found_dup_sack)
1665
		state.flag |= FLAG_DSACKING_ACK;
1666 1667 1668 1669 1670 1671 1672 1673

	/* Eliminate too old ACKs, but take into
	 * account more or less fresh ones, they can
	 * contain valid SACK info.
	 */
	if (before(TCP_SKB_CB(ack_skb)->ack_seq, prior_snd_una - tp->max_window))
		return 0;

1674 1675 1676
	if (!tp->packets_out)
		goto out;

1677 1678 1679
	used_sacks = 0;
	first_sack_index = 0;
	for (i = 0; i < num_sacks; i++) {
E
Eric Dumazet 已提交
1680
		bool dup_sack = !i && found_dup_sack;
1681

1682 1683
		sp[used_sacks].start_seq = get_unaligned_be32(&sp_wire[i].start_seq);
		sp[used_sacks].end_seq = get_unaligned_be32(&sp_wire[i].end_seq);
1684 1685 1686 1687

		if (!tcp_is_sackblock_valid(tp, dup_sack,
					    sp[used_sacks].start_seq,
					    sp[used_sacks].end_seq)) {
1688 1689
			int mib_idx;

1690 1691
			if (dup_sack) {
				if (!tp->undo_marker)
1692
					mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO;
1693
				else
1694
					mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD;
1695 1696 1697 1698 1699
			} else {
				/* Don't count olds caused by ACK reordering */
				if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) &&
				    !after(sp[used_sacks].end_seq, tp->snd_una))
					continue;
1700
				mib_idx = LINUX_MIB_TCPSACKDISCARD;
1701
			}
1702

1703
			NET_INC_STATS_BH(sock_net(sk), mib_idx);
1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715
			if (i == 0)
				first_sack_index = -1;
			continue;
		}

		/* Ignore very old stuff early */
		if (!after(sp[used_sacks].end_seq, prior_snd_una))
			continue;

		used_sacks++;
	}

1716 1717
	/* order SACK blocks to allow in order walk of the retrans queue */
	for (i = used_sacks - 1; i > 0; i--) {
1718 1719
		for (j = 0; j < i; j++) {
			if (after(sp[j].start_seq, sp[j + 1].start_seq)) {
1720
				swap(sp[j], sp[j + 1]);
1721

1722 1723
				/* Track where the first SACK block goes to */
				if (j == first_sack_index)
1724
					first_sack_index = j + 1;
1725 1726 1727 1728
			}
		}
	}

1729
	skb = tcp_write_queue_head(sk);
1730
	state.fack_count = 0;
1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741
	i = 0;

	if (!tp->sacked_out) {
		/* It's already past, so skip checking against it */
		cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
	} else {
		cache = tp->recv_sack_cache;
		/* Skip empty blocks in at head of the cache */
		while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq &&
		       !cache->end_seq)
			cache++;
1742 1743
	}

1744
	while (i < used_sacks) {
1745 1746
		u32 start_seq = sp[i].start_seq;
		u32 end_seq = sp[i].end_seq;
E
Eric Dumazet 已提交
1747
		bool dup_sack = (found_dup_sack && (i == first_sack_index));
1748
		struct tcp_sack_block *next_dup = NULL;
1749

1750 1751
		if (found_dup_sack && ((i + 1) == first_sack_index))
			next_dup = &sp[i + 1];
L
Linus Torvalds 已提交
1752

1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763
		/* Skip too early cached blocks */
		while (tcp_sack_cache_ok(tp, cache) &&
		       !before(start_seq, cache->end_seq))
			cache++;

		/* Can skip some work by looking recv_sack_cache? */
		if (tcp_sack_cache_ok(tp, cache) && !dup_sack &&
		    after(end_seq, cache->start_seq)) {

			/* Head todo? */
			if (before(start_seq, cache->start_seq)) {
1764 1765
				skb = tcp_sacktag_skip(skb, sk, &state,
						       start_seq);
1766
				skb = tcp_sacktag_walk(skb, sk, next_dup,
1767
						       &state,
1768 1769
						       start_seq,
						       cache->start_seq,
1770
						       dup_sack);
1771
			}
1772

1773
			/* Rest of the block already fully processed? */
1774
			if (!after(end_seq, cache->end_seq))
1775
				goto advance_sp;
1776

1777
			skb = tcp_maybe_skipping_dsack(skb, sk, next_dup,
1778 1779
						       &state,
						       cache->end_seq);
1780

1781
			/* ...tail remains todo... */
1782
			if (tcp_highest_sack_seq(tp) == cache->end_seq) {
1783
				/* ...but better entrypoint exists! */
1784 1785 1786
				skb = tcp_highest_sack(sk);
				if (skb == NULL)
					break;
1787
				state.fack_count = tp->fackets_out;
1788 1789
				cache++;
				goto walk;
1790 1791
			}

1792
			skb = tcp_sacktag_skip(skb, sk, &state, cache->end_seq);
1793 1794 1795 1796
			/* Check overlap against next cached too (past this one already) */
			cache++;
			continue;
		}
L
Linus Torvalds 已提交
1797

1798 1799 1800 1801
		if (!before(start_seq, tcp_highest_sack_seq(tp))) {
			skb = tcp_highest_sack(sk);
			if (skb == NULL)
				break;
1802
			state.fack_count = tp->fackets_out;
L
Linus Torvalds 已提交
1803
		}
1804
		skb = tcp_sacktag_skip(skb, sk, &state, start_seq);
1805 1806

walk:
1807 1808
		skb = tcp_sacktag_walk(skb, sk, next_dup, &state,
				       start_seq, end_seq, dup_sack);
1809

1810 1811
advance_sp:
		i++;
L
Linus Torvalds 已提交
1812 1813
	}

1814 1815 1816 1817 1818 1819 1820 1821
	/* Clear the head of the cache sack blocks so we can skip it next time */
	for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) {
		tp->recv_sack_cache[i].start_seq = 0;
		tp->recv_sack_cache[i].end_seq = 0;
	}
	for (j = 0; j < used_sacks; j++)
		tp->recv_sack_cache[i++] = sp[j];

1822
	tcp_mark_lost_retrans(sk);
L
Linus Torvalds 已提交
1823

1824 1825
	tcp_verify_left_out(tp);

1826
	if ((state.reord < tp->fackets_out) &&
Y
Yuchung Cheng 已提交
1827
	    ((inet_csk(sk)->icsk_ca_state != TCP_CA_Loss) || tp->undo_marker))
1828
		tcp_update_reordering(sk, tp->fackets_out - state.reord, 0);
L
Linus Torvalds 已提交
1829

1830 1831
out:

L
Linus Torvalds 已提交
1832
#if FASTRETRANS_DEBUG > 0
1833 1834 1835 1836
	WARN_ON((int)tp->sacked_out < 0);
	WARN_ON((int)tp->lost_out < 0);
	WARN_ON((int)tp->retrans_out < 0);
	WARN_ON((int)tcp_packets_in_flight(tp) < 0);
L
Linus Torvalds 已提交
1837
#endif
1838
	*sack_rtt_us = state.rtt_us;
1839
	return state.flag;
L
Linus Torvalds 已提交
1840 1841
}

1842
/* Limits sacked_out so that sum with lost_out isn't ever larger than
E
Eric Dumazet 已提交
1843
 * packets_out. Returns false if sacked_out adjustement wasn't necessary.
1844
 */
E
Eric Dumazet 已提交
1845
static bool tcp_limit_reno_sacked(struct tcp_sock *tp)
1846 1847 1848 1849 1850 1851 1852 1853
{
	u32 holes;

	holes = max(tp->lost_out, 1U);
	holes = min(holes, tp->packets_out);

	if ((tp->sacked_out + holes) > tp->packets_out) {
		tp->sacked_out = tp->packets_out - holes;
E
Eric Dumazet 已提交
1854
		return true;
1855
	}
E
Eric Dumazet 已提交
1856
	return false;
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867
}

/* If we receive more dupacks than we expected counting segments
 * in assumption of absent reordering, interpret this as reordering.
 * The only another reason could be bug in receiver TCP.
 */
static void tcp_check_reno_reordering(struct sock *sk, const int addend)
{
	struct tcp_sock *tp = tcp_sk(sk);
	if (tcp_limit_reno_sacked(tp))
		tcp_update_reordering(sk, tp->packets_out + addend, 0);
1868 1869 1870 1871 1872 1873 1874 1875 1876
}

/* Emulate SACKs for SACKless connection: account for a new dupack. */

static void tcp_add_reno_sack(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	tp->sacked_out++;
	tcp_check_reno_reordering(sk, 0);
1877
	tcp_verify_left_out(tp);
1878 1879 1880 1881 1882 1883 1884 1885 1886 1887
}

/* Account for ACK, ACKing some data in Reno Recovery phase. */

static void tcp_remove_reno_sacks(struct sock *sk, int acked)
{
	struct tcp_sock *tp = tcp_sk(sk);

	if (acked > 0) {
		/* One ACK acked hole. The rest eat duplicate ACKs. */
1888
		if (acked - 1 >= tp->sacked_out)
1889 1890
			tp->sacked_out = 0;
		else
1891
			tp->sacked_out -= acked - 1;
1892 1893
	}
	tcp_check_reno_reordering(sk, acked);
1894
	tcp_verify_left_out(tp);
1895 1896 1897 1898 1899 1900 1901
}

static inline void tcp_reset_reno_sack(struct tcp_sock *tp)
{
	tp->sacked_out = 0;
}

Y
Yuchung Cheng 已提交
1902
void tcp_clear_retrans(struct tcp_sock *tp)
L
Linus Torvalds 已提交
1903 1904 1905 1906
{
	tp->retrans_out = 0;
	tp->lost_out = 0;
	tp->undo_marker = 0;
Y
Yuchung Cheng 已提交
1907
	tp->undo_retrans = -1;
Y
Yuchung Cheng 已提交
1908 1909
	tp->fackets_out = 0;
	tp->sacked_out = 0;
L
Linus Torvalds 已提交
1910 1911
}

Y
Yuchung Cheng 已提交
1912
static inline void tcp_init_undo(struct tcp_sock *tp)
1913
{
Y
Yuchung Cheng 已提交
1914 1915 1916
	tp->undo_marker = tp->snd_una;
	/* Retransmission still in flight may cause DSACKs later. */
	tp->undo_retrans = tp->retrans_out ? : -1;
1917 1918
}

1919
/* Enter Loss state. If we detect SACK reneging, forget all SACK information
L
Linus Torvalds 已提交
1920 1921 1922
 * and reset tags completely, otherwise preserve SACKs. If receiver
 * dropped its ofo queue, we will know this due to reneging detection.
 */
1923
void tcp_enter_loss(struct sock *sk)
L
Linus Torvalds 已提交
1924
{
1925
	const struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
1926 1927
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb;
Y
Yuchung Cheng 已提交
1928
	bool new_recovery = false;
1929
	bool is_reneg;			/* is receiver reneging on SACKs? */
L
Linus Torvalds 已提交
1930 1931

	/* Reduce ssthresh if it has not yet been made inside this window. */
Y
Yuchung Cheng 已提交
1932 1933
	if (icsk->icsk_ca_state <= TCP_CA_Disorder ||
	    !after(tp->high_seq, tp->snd_una) ||
1934
	    (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) {
Y
Yuchung Cheng 已提交
1935
		new_recovery = true;
1936 1937 1938
		tp->prior_ssthresh = tcp_current_ssthresh(sk);
		tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
		tcp_ca_event(sk, CA_EVENT_LOSS);
Y
Yuchung Cheng 已提交
1939
		tcp_init_undo(tp);
L
Linus Torvalds 已提交
1940 1941 1942 1943 1944
	}
	tp->snd_cwnd	   = 1;
	tp->snd_cwnd_cnt   = 0;
	tp->snd_cwnd_stamp = tcp_time_stamp;

Y
Yuchung Cheng 已提交
1945 1946
	tp->retrans_out = 0;
	tp->lost_out = 0;
1947 1948 1949

	if (tcp_is_reno(tp))
		tcp_reset_reno_sack(tp);
L
Linus Torvalds 已提交
1950

1951 1952 1953 1954
	skb = tcp_write_queue_head(sk);
	is_reneg = skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED);
	if (is_reneg) {
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
1955 1956
		tp->sacked_out = 0;
		tp->fackets_out = 0;
1957
	}
1958
	tcp_clear_all_retrans_hints(tp);
L
Linus Torvalds 已提交
1959

1960 1961 1962
	tcp_for_write_queue(skb, sk) {
		if (skb == tcp_send_head(sk))
			break;
1963

L
Linus Torvalds 已提交
1964
		TCP_SKB_CB(skb)->sacked &= (~TCPCB_TAGBITS)|TCPCB_SACKED_ACKED;
1965
		if (!(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED) || is_reneg) {
L
Linus Torvalds 已提交
1966 1967 1968
			TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
			TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
			tp->lost_out += tcp_skb_pcount(skb);
1969
			tp->retransmit_high = TCP_SKB_CB(skb)->end_seq;
L
Linus Torvalds 已提交
1970 1971
		}
	}
1972
	tcp_verify_left_out(tp);
L
Linus Torvalds 已提交
1973

1974 1975 1976 1977 1978 1979 1980
	/* Timeout in disordered state after receiving substantial DUPACKs
	 * suggests that the degree of reordering is over-estimated.
	 */
	if (icsk->icsk_ca_state <= TCP_CA_Disorder &&
	    tp->sacked_out >= sysctl_tcp_reordering)
		tp->reordering = min_t(unsigned int, tp->reordering,
				       sysctl_tcp_reordering);
1981
	tcp_set_ca_state(sk, TCP_CA_Loss);
L
Linus Torvalds 已提交
1982
	tp->high_seq = tp->snd_nxt;
1983
	tcp_ecn_queue_cwr(tp);
Y
Yuchung Cheng 已提交
1984 1985 1986 1987 1988 1989 1990 1991

	/* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous
	 * loss recovery is underway except recurring timeout(s) on
	 * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing
	 */
	tp->frto = sysctl_tcp_frto &&
		   (new_recovery || icsk->icsk_retransmits) &&
		   !inet_csk(sk)->icsk_mtup.probe_size;
L
Linus Torvalds 已提交
1992 1993
}

1994 1995 1996 1997
/* If ACK arrived pointing to a remembered SACK, it means that our
 * remembered SACKs do not reflect real state of receiver i.e.
 * receiver _host_ is heavily congested (or buggy).
 *
1998 1999 2000 2001 2002
 * To avoid big spurious retransmission bursts due to transient SACK
 * scoreboard oddities that look like reneging, we give the receiver a
 * little time (max(RTT/2, 10ms)) to send us some more ACKs that will
 * restore sanity to the SACK scoreboard. If the apparent reneging
 * persists until this RTO then we'll clear the SACK scoreboard.
2003
 */
E
Eric Dumazet 已提交
2004
static bool tcp_check_sack_reneging(struct sock *sk, int flag)
L
Linus Torvalds 已提交
2005
{
2006
	if (flag & FLAG_SACK_RENEGING) {
2007 2008 2009
		struct tcp_sock *tp = tcp_sk(sk);
		unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4),
					  msecs_to_jiffies(10));
L
Linus Torvalds 已提交
2010

2011
		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2012
					  delay, TCP_RTO_MAX);
E
Eric Dumazet 已提交
2013
		return true;
L
Linus Torvalds 已提交
2014
	}
E
Eric Dumazet 已提交
2015
	return false;
L
Linus Torvalds 已提交
2016 2017
}

2018
static inline int tcp_fackets_out(const struct tcp_sock *tp)
L
Linus Torvalds 已提交
2019
{
2020
	return tcp_is_reno(tp) ? tp->sacked_out + 1 : tp->fackets_out;
L
Linus Torvalds 已提交
2021 2022
}

2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037
/* Heurestics to calculate number of duplicate ACKs. There's no dupACKs
 * counter when SACK is enabled (without SACK, sacked_out is used for
 * that purpose).
 *
 * Instead, with FACK TCP uses fackets_out that includes both SACKed
 * segments up to the highest received SACK block so far and holes in
 * between them.
 *
 * With reordering, holes may still be in flight, so RFC3517 recovery
 * uses pure sacked_out (total number of SACKed segments) even though
 * it violates the RFC that uses duplicate ACKs, often these are equal
 * but when e.g. out-of-window ACKs or packet duplication occurs,
 * they differ. Since neither occurs due to loss, TCP should really
 * ignore them.
 */
2038
static inline int tcp_dupack_heuristics(const struct tcp_sock *tp)
2039 2040 2041 2042
{
	return tcp_is_fack(tp) ? tp->fackets_out : tp->sacked_out + 1;
}

2043 2044 2045 2046 2047 2048 2049 2050 2051
static bool tcp_pause_early_retransmit(struct sock *sk, int flag)
{
	struct tcp_sock *tp = tcp_sk(sk);
	unsigned long delay;

	/* Delay early retransmit and entering fast recovery for
	 * max(RTT/4, 2msec) unless ack has ECE mark, no RTT samples
	 * available, or RTO is scheduled to fire first.
	 */
N
Nandita Dukkipati 已提交
2052
	if (sysctl_tcp_early_retrans < 2 || sysctl_tcp_early_retrans > 3 ||
2053
	    (flag & FLAG_ECE) || !tp->srtt_us)
2054 2055
		return false;

2056 2057 2058
	delay = max(usecs_to_jiffies(tp->srtt_us >> 5),
		    msecs_to_jiffies(2));

2059 2060 2061
	if (!time_after(inet_csk(sk)->icsk_timeout, (jiffies + delay)))
		return false;

N
Nandita Dukkipati 已提交
2062 2063
	inet_csk_reset_xmit_timer(sk, ICSK_TIME_EARLY_RETRANS, delay,
				  TCP_RTO_MAX);
2064 2065 2066
	return true;
}

L
Linus Torvalds 已提交
2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159
/* Linux NewReno/SACK/FACK/ECN state machine.
 * --------------------------------------
 *
 * "Open"	Normal state, no dubious events, fast path.
 * "Disorder"   In all the respects it is "Open",
 *		but requires a bit more attention. It is entered when
 *		we see some SACKs or dupacks. It is split of "Open"
 *		mainly to move some processing from fast path to slow one.
 * "CWR"	CWND was reduced due to some Congestion Notification event.
 *		It can be ECN, ICMP source quench, local device congestion.
 * "Recovery"	CWND was reduced, we are fast-retransmitting.
 * "Loss"	CWND was reduced due to RTO timeout or SACK reneging.
 *
 * tcp_fastretrans_alert() is entered:
 * - each incoming ACK, if state is not "Open"
 * - when arrived ACK is unusual, namely:
 *	* SACK
 *	* Duplicate ACK.
 *	* ECN ECE.
 *
 * Counting packets in flight is pretty simple.
 *
 *	in_flight = packets_out - left_out + retrans_out
 *
 *	packets_out is SND.NXT-SND.UNA counted in packets.
 *
 *	retrans_out is number of retransmitted segments.
 *
 *	left_out is number of segments left network, but not ACKed yet.
 *
 *		left_out = sacked_out + lost_out
 *
 *     sacked_out: Packets, which arrived to receiver out of order
 *		   and hence not ACKed. With SACKs this number is simply
 *		   amount of SACKed data. Even without SACKs
 *		   it is easy to give pretty reliable estimate of this number,
 *		   counting duplicate ACKs.
 *
 *       lost_out: Packets lost by network. TCP has no explicit
 *		   "loss notification" feedback from network (for now).
 *		   It means that this number can be only _guessed_.
 *		   Actually, it is the heuristics to predict lossage that
 *		   distinguishes different algorithms.
 *
 *	F.e. after RTO, when all the queue is considered as lost,
 *	lost_out = packets_out and in_flight = retrans_out.
 *
 *		Essentially, we have now two algorithms counting
 *		lost packets.
 *
 *		FACK: It is the simplest heuristics. As soon as we decided
 *		that something is lost, we decide that _all_ not SACKed
 *		packets until the most forward SACK are lost. I.e.
 *		lost_out = fackets_out - sacked_out and left_out = fackets_out.
 *		It is absolutely correct estimate, if network does not reorder
 *		packets. And it loses any connection to reality when reordering
 *		takes place. We use FACK by default until reordering
 *		is suspected on the path to this destination.
 *
 *		NewReno: when Recovery is entered, we assume that one segment
 *		is lost (classic Reno). While we are in Recovery and
 *		a partial ACK arrives, we assume that one more packet
 *		is lost (NewReno). This heuristics are the same in NewReno
 *		and SACK.
 *
 *  Imagine, that's all! Forget about all this shamanism about CWND inflation
 *  deflation etc. CWND is real congestion window, never inflated, changes
 *  only according to classic VJ rules.
 *
 * Really tricky (and requiring careful tuning) part of algorithm
 * is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue().
 * The first determines the moment _when_ we should reduce CWND and,
 * hence, slow down forward transmission. In fact, it determines the moment
 * when we decide that hole is caused by loss, rather than by a reorder.
 *
 * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill
 * holes, caused by lost packets.
 *
 * And the most logically complicated part of algorithm is undo
 * heuristics. We detect false retransmits due to both too early
 * fast retransmit (reordering) and underestimated RTO, analyzing
 * timestamps and D-SACKs. When we detect that some segments were
 * retransmitted by mistake and CWND reduction was wrong, we undo
 * window reduction and abort recovery phase. This logic is hidden
 * inside several functions named tcp_try_undo_<something>.
 */

/* This function decides, when we should leave Disordered state
 * and enter Recovery phase, reducing congestion window.
 *
 * Main question: may we further continue forward transmission
 * with the same cwnd?
 */
E
Eric Dumazet 已提交
2160
static bool tcp_time_to_recover(struct sock *sk, int flag)
L
Linus Torvalds 已提交
2161
{
2162
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
2163 2164 2165 2166
	__u32 packets_out;

	/* Trick#1: The loss is proven. */
	if (tp->lost_out)
E
Eric Dumazet 已提交
2167
		return true;
L
Linus Torvalds 已提交
2168 2169

	/* Not-A-Trick#2 : Classic rule... */
2170
	if (tcp_dupack_heuristics(tp) > tp->reordering)
E
Eric Dumazet 已提交
2171
		return true;
L
Linus Torvalds 已提交
2172 2173 2174 2175 2176 2177 2178

	/* Trick#4: It is still not OK... But will it be useful to delay
	 * recovery more?
	 */
	packets_out = tp->packets_out;
	if (packets_out <= tp->reordering &&
	    tp->sacked_out >= max_t(__u32, packets_out/2, sysctl_tcp_reordering) &&
2179
	    !tcp_may_send_now(sk)) {
L
Linus Torvalds 已提交
2180 2181 2182
		/* We have nothing to send. This connection is limited
		 * either by receiver window or by application.
		 */
E
Eric Dumazet 已提交
2183
		return true;
L
Linus Torvalds 已提交
2184 2185
	}

A
Andreas Petlund 已提交
2186 2187 2188 2189 2190 2191 2192 2193
	/* If a thin stream is detected, retransmit after first
	 * received dupack. Employ only if SACK is supported in order
	 * to avoid possible corner-case series of spurious retransmissions
	 * Use only if there are no unsent data.
	 */
	if ((tp->thin_dupack || sysctl_tcp_thin_dupack) &&
	    tcp_stream_is_thin(tp) && tcp_dupack_heuristics(tp) > 1 &&
	    tcp_is_sack(tp) && !tcp_send_head(sk))
E
Eric Dumazet 已提交
2194
		return true;
A
Andreas Petlund 已提交
2195

Y
Yuchung Cheng 已提交
2196 2197 2198 2199 2200 2201
	/* Trick#6: TCP early retransmit, per RFC5827.  To avoid spurious
	 * retransmissions due to small network reorderings, we implement
	 * Mitigation A.3 in the RFC and delay the retransmission for a short
	 * interval if appropriate.
	 */
	if (tp->do_early_retrans && !tp->retrans_out && tp->sacked_out &&
N
Nandita Dukkipati 已提交
2202
	    (tp->packets_out >= (tp->sacked_out + 1) && tp->packets_out < 4) &&
Y
Yuchung Cheng 已提交
2203
	    !tcp_may_send_now(sk))
2204
		return !tcp_pause_early_retransmit(sk, flag);
Y
Yuchung Cheng 已提交
2205

E
Eric Dumazet 已提交
2206
	return false;
L
Linus Torvalds 已提交
2207 2208
}

2209 2210 2211 2212 2213
/* Detect loss in event "A" above by marking head of queue up as lost.
 * For FACK or non-SACK(Reno) senders, the first "packets" number of segments
 * are considered lost. For RFC3517 SACK, a segment is considered lost if it
 * has at least tp->reordering SACKed seqments above it; "packets" refers to
 * the maximum SACKed segments to pass before reaching this limit.
2214
 */
I
Ilpo Järvinen 已提交
2215
static void tcp_mark_head_lost(struct sock *sk, int packets, int mark_head)
L
Linus Torvalds 已提交
2216
{
2217
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
2218
	struct sk_buff *skb;
2219 2220 2221
	int cnt, oldcnt;
	int err;
	unsigned int mss;
2222 2223
	/* Use SACK to deduce losses of new sequences sent during recovery */
	const u32 loss_high = tcp_is_sack(tp) ?  tp->snd_nxt : tp->high_seq;
L
Linus Torvalds 已提交
2224

2225
	WARN_ON(packets > tp->packets_out);
2226 2227 2228
	if (tp->lost_skb_hint) {
		skb = tp->lost_skb_hint;
		cnt = tp->lost_cnt_hint;
I
Ilpo Järvinen 已提交
2229 2230 2231
		/* Head already handled? */
		if (mark_head && skb != tcp_write_queue_head(sk))
			return;
2232
	} else {
2233
		skb = tcp_write_queue_head(sk);
2234 2235
		cnt = 0;
	}
L
Linus Torvalds 已提交
2236

2237 2238 2239
	tcp_for_write_queue_from(skb, sk) {
		if (skb == tcp_send_head(sk))
			break;
2240 2241 2242 2243
		/* TODO: do this better */
		/* this is not the most efficient way to do this... */
		tp->lost_skb_hint = skb;
		tp->lost_cnt_hint = cnt;
2244

2245
		if (after(TCP_SKB_CB(skb)->end_seq, loss_high))
2246 2247 2248
			break;

		oldcnt = cnt;
2249
		if (tcp_is_fack(tp) || tcp_is_reno(tp) ||
2250 2251 2252
		    (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
			cnt += tcp_skb_pcount(skb);

2253
		if (cnt > packets) {
2254
			if ((tcp_is_sack(tp) && !tcp_is_fack(tp)) ||
2255
			    (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) ||
2256
			    (oldcnt >= packets))
2257 2258 2259
				break;

			mss = skb_shinfo(skb)->gso_size;
2260 2261
			err = tcp_fragment(sk, skb, (packets - oldcnt) * mss,
					   mss, GFP_ATOMIC);
2262 2263 2264 2265 2266
			if (err < 0)
				break;
			cnt = packets;
		}

2267
		tcp_skb_mark_lost(tp, skb);
I
Ilpo Järvinen 已提交
2268 2269 2270

		if (mark_head)
			break;
L
Linus Torvalds 已提交
2271
	}
2272
	tcp_verify_left_out(tp);
L
Linus Torvalds 已提交
2273 2274 2275 2276
}

/* Account newly detected lost packet(s) */

2277
static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit)
L
Linus Torvalds 已提交
2278
{
2279 2280
	struct tcp_sock *tp = tcp_sk(sk);

2281
	if (tcp_is_reno(tp)) {
I
Ilpo Järvinen 已提交
2282
		tcp_mark_head_lost(sk, 1, 1);
2283
	} else if (tcp_is_fack(tp)) {
L
Linus Torvalds 已提交
2284 2285 2286
		int lost = tp->fackets_out - tp->reordering;
		if (lost <= 0)
			lost = 1;
I
Ilpo Järvinen 已提交
2287
		tcp_mark_head_lost(sk, lost, 0);
L
Linus Torvalds 已提交
2288
	} else {
2289
		int sacked_upto = tp->sacked_out - tp->reordering;
I
Ilpo Järvinen 已提交
2290 2291 2292 2293
		if (sacked_upto >= 0)
			tcp_mark_head_lost(sk, sacked_upto, 0);
		else if (fast_rexmit)
			tcp_mark_head_lost(sk, 1, 1);
L
Linus Torvalds 已提交
2294 2295 2296 2297 2298 2299 2300 2301 2302
	}
}

/* CWND moderation, preventing bursts due to too big ACKs
 * in dubious situations.
 */
static inline void tcp_moderate_cwnd(struct tcp_sock *tp)
{
	tp->snd_cwnd = min(tp->snd_cwnd,
2303
			   tcp_packets_in_flight(tp) + tcp_max_burst(tp));
L
Linus Torvalds 已提交
2304 2305 2306 2307 2308 2309
	tp->snd_cwnd_stamp = tcp_time_stamp;
}

/* Nothing was retransmitted or returned timestamp is less
 * than timestamp of the first retransmission.
 */
2310
static inline bool tcp_packet_delayed(const struct tcp_sock *tp)
L
Linus Torvalds 已提交
2311 2312 2313
{
	return !tp->retrans_stamp ||
		(tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
2314
		 before(tp->rx_opt.rcv_tsecr, tp->retrans_stamp));
L
Linus Torvalds 已提交
2315 2316 2317 2318
}

/* Undo procedures. */

2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347
/* We can clear retrans_stamp when there are no retransmissions in the
 * window. It would seem that it is trivially available for us in
 * tp->retrans_out, however, that kind of assumptions doesn't consider
 * what will happen if errors occur when sending retransmission for the
 * second time. ...It could the that such segment has only
 * TCPCB_EVER_RETRANS set at the present time. It seems that checking
 * the head skb is enough except for some reneging corner cases that
 * are not worth the effort.
 *
 * Main reason for all this complexity is the fact that connection dying
 * time now depends on the validity of the retrans_stamp, in particular,
 * that successive retransmissions of a segment must not advance
 * retrans_stamp under any conditions.
 */
static bool tcp_any_retrans_done(const struct sock *sk)
{
	const struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb;

	if (tp->retrans_out)
		return true;

	skb = tcp_write_queue_head(sk);
	if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS))
		return true;

	return false;
}

L
Linus Torvalds 已提交
2348
#if FASTRETRANS_DEBUG > 1
2349
static void DBGUNDO(struct sock *sk, const char *msg)
L
Linus Torvalds 已提交
2350
{
2351
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
2352
	struct inet_sock *inet = inet_sk(sk);
2353

2354
	if (sk->sk_family == AF_INET) {
2355 2356 2357 2358 2359 2360
		pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n",
			 msg,
			 &inet->inet_daddr, ntohs(inet->inet_dport),
			 tp->snd_cwnd, tcp_left_out(tp),
			 tp->snd_ssthresh, tp->prior_ssthresh,
			 tp->packets_out);
2361
	}
E
Eric Dumazet 已提交
2362
#if IS_ENABLED(CONFIG_IPV6)
2363 2364
	else if (sk->sk_family == AF_INET6) {
		struct ipv6_pinfo *np = inet6_sk(sk);
2365 2366 2367 2368 2369 2370
		pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n",
			 msg,
			 &np->daddr, ntohs(inet->inet_dport),
			 tp->snd_cwnd, tcp_left_out(tp),
			 tp->snd_ssthresh, tp->prior_ssthresh,
			 tp->packets_out);
2371 2372
	}
#endif
L
Linus Torvalds 已提交
2373 2374 2375 2376 2377
}
#else
#define DBGUNDO(x...) do { } while (0)
#endif

2378
static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss)
L
Linus Torvalds 已提交
2379
{
2380 2381
	struct tcp_sock *tp = tcp_sk(sk);

Y
Yuchung Cheng 已提交
2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
	if (unmark_loss) {
		struct sk_buff *skb;

		tcp_for_write_queue(skb, sk) {
			if (skb == tcp_send_head(sk))
				break;
			TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST;
		}
		tp->lost_out = 0;
		tcp_clear_all_retrans_hints(tp);
	}

L
Linus Torvalds 已提交
2394
	if (tp->prior_ssthresh) {
2395 2396 2397 2398
		const struct inet_connection_sock *icsk = inet_csk(sk);

		if (icsk->icsk_ca_ops->undo_cwnd)
			tp->snd_cwnd = icsk->icsk_ca_ops->undo_cwnd(sk);
L
Linus Torvalds 已提交
2399
		else
2400
			tp->snd_cwnd = max(tp->snd_cwnd, tp->snd_ssthresh << 1);
L
Linus Torvalds 已提交
2401

2402
		if (tp->prior_ssthresh > tp->snd_ssthresh) {
L
Linus Torvalds 已提交
2403
			tp->snd_ssthresh = tp->prior_ssthresh;
2404
			tcp_ecn_withdraw_cwr(tp);
L
Linus Torvalds 已提交
2405 2406 2407 2408 2409
		}
	} else {
		tp->snd_cwnd = max(tp->snd_cwnd, tp->snd_ssthresh);
	}
	tp->snd_cwnd_stamp = tcp_time_stamp;
2410
	tp->undo_marker = 0;
L
Linus Torvalds 已提交
2411 2412
}

2413
static inline bool tcp_may_undo(const struct tcp_sock *tp)
L
Linus Torvalds 已提交
2414
{
2415
	return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp));
L
Linus Torvalds 已提交
2416 2417 2418
}

/* People celebrate: "We love our President!" */
E
Eric Dumazet 已提交
2419
static bool tcp_try_undo_recovery(struct sock *sk)
L
Linus Torvalds 已提交
2420
{
2421 2422
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
2423
	if (tcp_may_undo(tp)) {
2424 2425
		int mib_idx;

L
Linus Torvalds 已提交
2426 2427 2428
		/* Happy end! We did not retransmit anything
		 * or our original transmission succeeded.
		 */
2429
		DBGUNDO(sk, inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans");
2430
		tcp_undo_cwnd_reduction(sk, false);
2431
		if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
2432
			mib_idx = LINUX_MIB_TCPLOSSUNDO;
L
Linus Torvalds 已提交
2433
		else
2434 2435
			mib_idx = LINUX_MIB_TCPFULLUNDO;

2436
		NET_INC_STATS_BH(sock_net(sk), mib_idx);
L
Linus Torvalds 已提交
2437
	}
2438
	if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) {
L
Linus Torvalds 已提交
2439 2440 2441 2442
		/* Hold old state until something *above* high_seq
		 * is ACKed. For Reno it is MUST to prevent false
		 * fast retransmits (RFC2582). SACK TCP is safe. */
		tcp_moderate_cwnd(tp);
2443 2444
		if (!tcp_any_retrans_done(sk))
			tp->retrans_stamp = 0;
E
Eric Dumazet 已提交
2445
		return true;
L
Linus Torvalds 已提交
2446
	}
2447
	tcp_set_ca_state(sk, TCP_CA_Open);
E
Eric Dumazet 已提交
2448
	return false;
L
Linus Torvalds 已提交
2449 2450 2451
}

/* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */
2452
static bool tcp_try_undo_dsack(struct sock *sk)
L
Linus Torvalds 已提交
2453
{
2454 2455
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
2456
	if (tp->undo_marker && !tp->undo_retrans) {
2457
		DBGUNDO(sk, "D-SACK");
2458
		tcp_undo_cwnd_reduction(sk, false);
2459
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPDSACKUNDO);
2460
		return true;
L
Linus Torvalds 已提交
2461
	}
2462
	return false;
L
Linus Torvalds 已提交
2463 2464
}

Y
Yuchung Cheng 已提交
2465 2466
/* Undo during loss recovery after partial ACK or using F-RTO. */
static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo)
L
Linus Torvalds 已提交
2467
{
2468 2469
	struct tcp_sock *tp = tcp_sk(sk);

Y
Yuchung Cheng 已提交
2470
	if (frto_undo || tcp_may_undo(tp)) {
2471
		tcp_undo_cwnd_reduction(sk, true);
2472

2473
		DBGUNDO(sk, "partial loss");
2474
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPLOSSUNDO);
Y
Yuchung Cheng 已提交
2475 2476 2477
		if (frto_undo)
			NET_INC_STATS_BH(sock_net(sk),
					 LINUX_MIB_TCPSPURIOUSRTOS);
2478
		inet_csk(sk)->icsk_retransmits = 0;
Y
Yuchung Cheng 已提交
2479
		if (frto_undo || tcp_is_sack(tp))
2480
			tcp_set_ca_state(sk, TCP_CA_Open);
E
Eric Dumazet 已提交
2481
		return true;
L
Linus Torvalds 已提交
2482
	}
E
Eric Dumazet 已提交
2483
	return false;
L
Linus Torvalds 已提交
2484 2485
}

2486 2487
/* The cwnd reduction in CWR and Recovery use the PRR algorithm
 * https://datatracker.ietf.org/doc/draft-ietf-tcpm-proportional-rate-reduction/
2488 2489 2490 2491 2492 2493 2494 2495
 * It computes the number of packets to send (sndcnt) based on packets newly
 * delivered:
 *   1) If the packets in flight is larger than ssthresh, PRR spreads the
 *	cwnd reductions across a full RTT.
 *   2) If packets in flight is lower than ssthresh (such as due to excess
 *	losses and/or application stalls), do not perform any further cwnd
 *	reductions, but instead slow start up to ssthresh.
 */
2496
static void tcp_init_cwnd_reduction(struct sock *sk)
2497 2498 2499 2500
{
	struct tcp_sock *tp = tcp_sk(sk);

	tp->high_seq = tp->snd_nxt;
N
Nandita Dukkipati 已提交
2501
	tp->tlp_high_seq = 0;
2502 2503 2504 2505
	tp->snd_cwnd_cnt = 0;
	tp->prior_cwnd = tp->snd_cwnd;
	tp->prr_delivered = 0;
	tp->prr_out = 0;
2506
	tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk);
2507
	tcp_ecn_queue_cwr(tp);
2508 2509
}

2510
static void tcp_cwnd_reduction(struct sock *sk, const int prior_unsacked,
2511
			       int fast_rexmit)
2512 2513 2514 2515
{
	struct tcp_sock *tp = tcp_sk(sk);
	int sndcnt = 0;
	int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp);
2516 2517
	int newly_acked_sacked = prior_unsacked -
				 (tp->packets_out - tp->sacked_out);
2518

2519
	tp->prr_delivered += newly_acked_sacked;
2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533
	if (tcp_packets_in_flight(tp) > tp->snd_ssthresh) {
		u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered +
			       tp->prior_cwnd - 1;
		sndcnt = div_u64(dividend, tp->prior_cwnd) - tp->prr_out;
	} else {
		sndcnt = min_t(int, delta,
			       max_t(int, tp->prr_delivered - tp->prr_out,
				     newly_acked_sacked) + 1);
	}

	sndcnt = max(sndcnt, (fast_rexmit ? 1 : 0));
	tp->snd_cwnd = tcp_packets_in_flight(tp) + sndcnt;
}

2534
static inline void tcp_end_cwnd_reduction(struct sock *sk)
L
Linus Torvalds 已提交
2535
{
2536
	struct tcp_sock *tp = tcp_sk(sk);
2537

2538 2539 2540 2541 2542
	/* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
	if (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR ||
	    (tp->undo_marker && tp->snd_ssthresh < TCP_INFINITE_SSTHRESH)) {
		tp->snd_cwnd = tp->snd_ssthresh;
		tp->snd_cwnd_stamp = tcp_time_stamp;
2543
	}
2544
	tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR);
L
Linus Torvalds 已提交
2545 2546
}

2547
/* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */
2548
void tcp_enter_cwr(struct sock *sk)
Y
Yuchung Cheng 已提交
2549 2550 2551 2552
{
	struct tcp_sock *tp = tcp_sk(sk);

	tp->prior_ssthresh = 0;
2553
	if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) {
Y
Yuchung Cheng 已提交
2554
		tp->undo_marker = 0;
2555
		tcp_init_cwnd_reduction(sk);
Y
Yuchung Cheng 已提交
2556 2557 2558 2559
		tcp_set_ca_state(sk, TCP_CA_CWR);
	}
}

2560 2561 2562 2563 2564
static void tcp_try_keep_open(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	int state = TCP_CA_Open;

2565
	if (tcp_left_out(tp) || tcp_any_retrans_done(sk))
2566 2567 2568 2569 2570 2571 2572 2573
		state = TCP_CA_Disorder;

	if (inet_csk(sk)->icsk_ca_state != state) {
		tcp_set_ca_state(sk, state);
		tp->high_seq = tp->snd_nxt;
	}
}

2574
static void tcp_try_to_open(struct sock *sk, int flag, const int prior_unsacked)
L
Linus Torvalds 已提交
2575
{
2576 2577
	struct tcp_sock *tp = tcp_sk(sk);

2578 2579
	tcp_verify_left_out(tp);

Y
Yuchung Cheng 已提交
2580
	if (!tcp_any_retrans_done(sk))
L
Linus Torvalds 已提交
2581 2582
		tp->retrans_stamp = 0;

2583
	if (flag & FLAG_ECE)
2584
		tcp_enter_cwr(sk);
L
Linus Torvalds 已提交
2585

2586
	if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) {
2587
		tcp_try_keep_open(sk);
L
Linus Torvalds 已提交
2588
	} else {
2589
		tcp_cwnd_reduction(sk, prior_unsacked, 0);
L
Linus Torvalds 已提交
2590 2591 2592
	}
}

J
John Heffner 已提交
2593 2594 2595 2596 2597 2598 2599 2600
static void tcp_mtup_probe_failed(struct sock *sk)
{
	struct inet_connection_sock *icsk = inet_csk(sk);

	icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1;
	icsk->icsk_mtup.probe_size = 0;
}

2601
static void tcp_mtup_probe_success(struct sock *sk)
J
John Heffner 已提交
2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct inet_connection_sock *icsk = inet_csk(sk);

	/* FIXME: breaks with very large cwnd */
	tp->prior_ssthresh = tcp_current_ssthresh(sk);
	tp->snd_cwnd = tp->snd_cwnd *
		       tcp_mss_to_mtu(sk, tp->mss_cache) /
		       icsk->icsk_mtup.probe_size;
	tp->snd_cwnd_cnt = 0;
	tp->snd_cwnd_stamp = tcp_time_stamp;
2613
	tp->snd_ssthresh = tcp_current_ssthresh(sk);
J
John Heffner 已提交
2614 2615 2616 2617 2618 2619

	icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size;
	icsk->icsk_mtup.probe_size = 0;
	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
}

2620 2621 2622 2623 2624 2625 2626 2627 2628
/* Do a simple retransmit without using the backoff mechanisms in
 * tcp_timer. This is used for path mtu discovery.
 * The socket is already locked here.
 */
void tcp_simple_retransmit(struct sock *sk)
{
	const struct inet_connection_sock *icsk = inet_csk(sk);
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb;
I
Ilpo Järvinen 已提交
2629
	unsigned int mss = tcp_current_mss(sk);
2630 2631 2632 2633 2634
	u32 prior_lost = tp->lost_out;

	tcp_for_write_queue(skb, sk) {
		if (skb == tcp_send_head(sk))
			break;
2635
		if (tcp_skb_seglen(skb) > mss &&
2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668
		    !(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) {
			if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
				TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
				tp->retrans_out -= tcp_skb_pcount(skb);
			}
			tcp_skb_mark_lost_uncond_verify(tp, skb);
		}
	}

	tcp_clear_retrans_hints_partial(tp);

	if (prior_lost == tp->lost_out)
		return;

	if (tcp_is_reno(tp))
		tcp_limit_reno_sacked(tp);

	tcp_verify_left_out(tp);

	/* Don't muck with the congestion window here.
	 * Reason is that we do not increase amount of _data_
	 * in network, but units changed and effective
	 * cwnd/ssthresh really reduced now.
	 */
	if (icsk->icsk_ca_state != TCP_CA_Loss) {
		tp->high_seq = tp->snd_nxt;
		tp->snd_ssthresh = tcp_current_ssthresh(sk);
		tp->prior_ssthresh = 0;
		tp->undo_marker = 0;
		tcp_set_ca_state(sk, TCP_CA_Loss);
	}
	tcp_xmit_retransmit_queue(sk);
}
E
Eric Dumazet 已提交
2669
EXPORT_SYMBOL(tcp_simple_retransmit);
2670

2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683
static void tcp_enter_recovery(struct sock *sk, bool ece_ack)
{
	struct tcp_sock *tp = tcp_sk(sk);
	int mib_idx;

	if (tcp_is_reno(tp))
		mib_idx = LINUX_MIB_TCPRENORECOVERY;
	else
		mib_idx = LINUX_MIB_TCPSACKRECOVERY;

	NET_INC_STATS_BH(sock_net(sk), mib_idx);

	tp->prior_ssthresh = 0;
Y
Yuchung Cheng 已提交
2684
	tcp_init_undo(tp);
2685 2686 2687 2688

	if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) {
		if (!ece_ack)
			tp->prior_ssthresh = tcp_current_ssthresh(sk);
2689
		tcp_init_cwnd_reduction(sk);
2690 2691 2692 2693
	}
	tcp_set_ca_state(sk, TCP_CA_Recovery);
}

2694 2695 2696
/* Process an ACK in CA_Loss state. Move to CA_Open if lost data are
 * recovered or spurious. Otherwise retransmits more on partial ACKs.
 */
Y
Yuchung Cheng 已提交
2697
static void tcp_process_loss(struct sock *sk, int flag, bool is_dupack)
2698 2699
{
	struct tcp_sock *tp = tcp_sk(sk);
Y
Yuchung Cheng 已提交
2700
	bool recovered = !before(tp->snd_una, tp->high_seq);
2701

Y
Yuchung Cheng 已提交
2702
	if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */
2703 2704 2705 2706
		/* Step 3.b. A timeout is spurious if not all data are
		 * lost, i.e., never-retransmitted data are (s)acked.
		 */
		if (tcp_try_undo_loss(sk, flag & FLAG_ORIG_SACK_ACKED))
Y
Yuchung Cheng 已提交
2707
			return;
2708

Y
Yuchung Cheng 已提交
2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723
		if (after(tp->snd_nxt, tp->high_seq) &&
		    (flag & FLAG_DATA_SACKED || is_dupack)) {
			tp->frto = 0; /* Loss was real: 2nd part of step 3.a */
		} else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) {
			tp->high_seq = tp->snd_nxt;
			__tcp_push_pending_frames(sk, tcp_current_mss(sk),
						  TCP_NAGLE_OFF);
			if (after(tp->snd_nxt, tp->high_seq))
				return; /* Step 2.b */
			tp->frto = 0;
		}
	}

	if (recovered) {
		/* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */
2724 2725 2726
		tcp_try_undo_recovery(sk);
		return;
	}
Y
Yuchung Cheng 已提交
2727 2728 2729 2730 2731 2732 2733 2734 2735 2736
	if (tcp_is_reno(tp)) {
		/* A Reno DUPACK means new data in F-RTO step 2.b above are
		 * delivered. Lower inflight to clock out (re)tranmissions.
		 */
		if (after(tp->snd_nxt, tp->high_seq) && is_dupack)
			tcp_add_reno_sack(sk);
		else if (flag & FLAG_SND_UNA_ADVANCED)
			tcp_reset_reno_sack(tp);
	}
	if (tcp_try_undo_loss(sk, false))
2737 2738 2739 2740
		return;
	tcp_xmit_retransmit_queue(sk);
}

Y
Yuchung Cheng 已提交
2741
/* Undo during fast recovery after partial ACK. */
2742 2743
static bool tcp_try_undo_partial(struct sock *sk, const int acked,
				 const int prior_unsacked)
Y
Yuchung Cheng 已提交
2744 2745 2746
{
	struct tcp_sock *tp = tcp_sk(sk);

2747
	if (tp->undo_marker && tcp_packet_delayed(tp)) {
Y
Yuchung Cheng 已提交
2748 2749 2750
		/* Plain luck! Hole if filled with delayed
		 * packet, rather than with a retransmit.
		 */
2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762
		tcp_update_reordering(sk, tcp_fackets_out(tp) + acked, 1);

		/* We are getting evidence that the reordering degree is higher
		 * than we realized. If there are no retransmits out then we
		 * can undo. Otherwise we clock out new packets but do not
		 * mark more packets lost or retransmit more.
		 */
		if (tp->retrans_out) {
			tcp_cwnd_reduction(sk, prior_unsacked, 0);
			return true;
		}

Y
Yuchung Cheng 已提交
2763 2764 2765
		if (!tcp_any_retrans_done(sk))
			tp->retrans_stamp = 0;

2766 2767
		DBGUNDO(sk, "partial recovery");
		tcp_undo_cwnd_reduction(sk, true);
Y
Yuchung Cheng 已提交
2768
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO);
2769 2770
		tcp_try_keep_open(sk);
		return true;
Y
Yuchung Cheng 已提交
2771
	}
2772
	return false;
Y
Yuchung Cheng 已提交
2773 2774
}

L
Linus Torvalds 已提交
2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785
/* Process an event, which can update packets-in-flight not trivially.
 * Main goal of this function is to calculate new estimate for left_out,
 * taking into account both packets sitting in receiver's buffer and
 * packets lost by network.
 *
 * Besides that it does CWND reduction, when packet loss is detected
 * and changes state of machine.
 *
 * It does _not_ decide what to send, it is made in function
 * tcp_xmit_retransmit_queue().
 */
2786 2787
static void tcp_fastretrans_alert(struct sock *sk, const int acked,
				  const int prior_unsacked,
2788
				  bool is_dupack, int flag)
L
Linus Torvalds 已提交
2789
{
2790
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
2791
	struct tcp_sock *tp = tcp_sk(sk);
Y
Yuchung Cheng 已提交
2792
	bool do_lost = is_dupack || ((flag & FLAG_DATA_SACKED) &&
2793
				    (tcp_fackets_out(tp) > tp->reordering));
2794
	int fast_rexmit = 0;
L
Linus Torvalds 已提交
2795

2796
	if (WARN_ON(!tp->packets_out && tp->sacked_out))
L
Linus Torvalds 已提交
2797
		tp->sacked_out = 0;
I
Ilpo Järvinen 已提交
2798
	if (WARN_ON(!tp->sacked_out && tp->fackets_out))
L
Linus Torvalds 已提交
2799 2800
		tp->fackets_out = 0;

2801
	/* Now state machine starts.
L
Linus Torvalds 已提交
2802
	 * A. ECE, hence prohibit cwnd undoing, the reduction is required. */
2803
	if (flag & FLAG_ECE)
L
Linus Torvalds 已提交
2804 2805 2806
		tp->prior_ssthresh = 0;

	/* B. In all the states check for reneging SACKs. */
2807
	if (tcp_check_sack_reneging(sk, flag))
L
Linus Torvalds 已提交
2808 2809
		return;

2810
	/* C. Check consistency of the current state. */
2811
	tcp_verify_left_out(tp);
L
Linus Torvalds 已提交
2812

2813
	/* D. Check state exit conditions. State can be terminated
L
Linus Torvalds 已提交
2814
	 *    when high_seq is ACKed. */
2815
	if (icsk->icsk_ca_state == TCP_CA_Open) {
2816
		WARN_ON(tp->retrans_out != 0);
L
Linus Torvalds 已提交
2817 2818
		tp->retrans_stamp = 0;
	} else if (!before(tp->snd_una, tp->high_seq)) {
2819
		switch (icsk->icsk_ca_state) {
L
Linus Torvalds 已提交
2820 2821 2822 2823
		case TCP_CA_CWR:
			/* CWR is to be held something *above* high_seq
			 * is ACKed for CWR bit to reach receiver. */
			if (tp->snd_una != tp->high_seq) {
2824
				tcp_end_cwnd_reduction(sk);
2825
				tcp_set_ca_state(sk, TCP_CA_Open);
L
Linus Torvalds 已提交
2826 2827 2828 2829
			}
			break;

		case TCP_CA_Recovery:
2830
			if (tcp_is_reno(tp))
L
Linus Torvalds 已提交
2831
				tcp_reset_reno_sack(tp);
2832
			if (tcp_try_undo_recovery(sk))
L
Linus Torvalds 已提交
2833
				return;
2834
			tcp_end_cwnd_reduction(sk);
L
Linus Torvalds 已提交
2835 2836 2837 2838
			break;
		}
	}

2839
	/* E. Process state. */
2840
	switch (icsk->icsk_ca_state) {
L
Linus Torvalds 已提交
2841
	case TCP_CA_Recovery:
2842
		if (!(flag & FLAG_SND_UNA_ADVANCED)) {
2843
			if (tcp_is_reno(tp) && is_dupack)
2844
				tcp_add_reno_sack(sk);
2845 2846 2847 2848 2849 2850 2851
		} else {
			if (tcp_try_undo_partial(sk, acked, prior_unsacked))
				return;
			/* Partial ACK arrived. Force fast retransmit. */
			do_lost = tcp_is_reno(tp) ||
				  tcp_fackets_out(tp) > tp->reordering;
		}
2852 2853 2854 2855
		if (tcp_try_undo_dsack(sk)) {
			tcp_try_keep_open(sk);
			return;
		}
L
Linus Torvalds 已提交
2856 2857
		break;
	case TCP_CA_Loss:
Y
Yuchung Cheng 已提交
2858
		tcp_process_loss(sk, flag, is_dupack);
2859
		if (icsk->icsk_ca_state != TCP_CA_Open)
L
Linus Torvalds 已提交
2860
			return;
2861
		/* Fall through to processing in Open state. */
L
Linus Torvalds 已提交
2862
	default:
2863
		if (tcp_is_reno(tp)) {
2864
			if (flag & FLAG_SND_UNA_ADVANCED)
L
Linus Torvalds 已提交
2865 2866
				tcp_reset_reno_sack(tp);
			if (is_dupack)
2867
				tcp_add_reno_sack(sk);
L
Linus Torvalds 已提交
2868 2869
		}

2870
		if (icsk->icsk_ca_state <= TCP_CA_Disorder)
2871
			tcp_try_undo_dsack(sk);
L
Linus Torvalds 已提交
2872

2873
		if (!tcp_time_to_recover(sk, flag)) {
2874
			tcp_try_to_open(sk, flag, prior_unsacked);
L
Linus Torvalds 已提交
2875 2876 2877
			return;
		}

J
John Heffner 已提交
2878 2879 2880
		/* MTU probe failure: don't reduce cwnd */
		if (icsk->icsk_ca_state < TCP_CA_CWR &&
		    icsk->icsk_mtup.probe_size &&
2881
		    tp->snd_una == tp->mtu_probe.probe_seq_start) {
J
John Heffner 已提交
2882 2883 2884 2885 2886 2887 2888
			tcp_mtup_probe_failed(sk);
			/* Restores the reduction we did in tcp_mtup_probe() */
			tp->snd_cwnd++;
			tcp_simple_retransmit(sk);
			return;
		}

L
Linus Torvalds 已提交
2889
		/* Otherwise enter Recovery state */
2890
		tcp_enter_recovery(sk, (flag & FLAG_ECE));
2891
		fast_rexmit = 1;
L
Linus Torvalds 已提交
2892 2893
	}

2894
	if (do_lost)
2895
		tcp_update_scoreboard(sk, fast_rexmit);
2896
	tcp_cwnd_reduction(sk, prior_unsacked, fast_rexmit);
L
Linus Torvalds 已提交
2897 2898 2899
	tcp_xmit_retransmit_queue(sk);
}

Y
Yuchung Cheng 已提交
2900
static inline bool tcp_ack_update_rtt(struct sock *sk, const int flag,
2901
				      long seq_rtt_us, long sack_rtt_us)
2902
{
2903 2904 2905 2906 2907 2908 2909 2910
	const struct tcp_sock *tp = tcp_sk(sk);

	/* Prefer RTT measured from ACK's timing to TS-ECR. This is because
	 * broken middle-boxes or peers may corrupt TS-ECR fields. But
	 * Karn's algorithm forbids taking RTT if some retransmitted data
	 * is acked (RFC6298).
	 */
	if (flag & FLAG_RETRANS_DATA_ACKED)
2911
		seq_rtt_us = -1L;
2912

2913 2914
	if (seq_rtt_us < 0)
		seq_rtt_us = sack_rtt_us;
Y
Yuchung Cheng 已提交
2915

L
Linus Torvalds 已提交
2916 2917 2918 2919 2920 2921
	/* RTTM Rule: A TSecr value received in a segment is used to
	 * update the averaged RTT measurement only if the segment
	 * acknowledges some new data, i.e., only if it advances the
	 * left edge of the send window.
	 * See draft-ietf-tcplw-high-performance-00, section 3.3.
	 */
2922
	if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
2923
	    flag & FLAG_ACKED)
2924
		seq_rtt_us = jiffies_to_usecs(tcp_time_stamp - tp->rx_opt.rcv_tsecr);
2925

2926
	if (seq_rtt_us < 0)
Y
Yuchung Cheng 已提交
2927
		return false;
L
Linus Torvalds 已提交
2928

2929
	tcp_rtt_estimator(sk, seq_rtt_us);
2930
	tcp_set_rto(sk);
L
Linus Torvalds 已提交
2931

2932 2933
	/* RFC6298: only reset backoff on valid RTT measurement. */
	inet_csk(sk)->icsk_backoff = 0;
Y
Yuchung Cheng 已提交
2934
	return true;
L
Linus Torvalds 已提交
2935 2936
}

2937
/* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */
2938
static void tcp_synack_rtt_meas(struct sock *sk, const u32 synack_stamp)
2939 2940
{
	struct tcp_sock *tp = tcp_sk(sk);
2941
	long seq_rtt_us = -1L;
2942

2943
	if (synack_stamp && !tp->total_retrans)
2944
		seq_rtt_us = jiffies_to_usecs(tcp_time_stamp - synack_stamp);
2945 2946 2947 2948

	/* If the ACK acks both the SYNACK and the (Fast Open'd) data packets
	 * sent in SYN_RECV, SYNACK RTT is the smooth RTT computed in tcp_ack()
	 */
2949 2950
	if (!tp->srtt_us)
		tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, seq_rtt_us, -1L);
2951 2952
}

2953
static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
L
Linus Torvalds 已提交
2954
{
2955
	const struct inet_connection_sock *icsk = inet_csk(sk);
2956 2957

	icsk->icsk_ca_ops->cong_avoid(sk, ack, acked);
2958
	tcp_sk(sk)->snd_cwnd_stamp = tcp_time_stamp;
L
Linus Torvalds 已提交
2959 2960 2961 2962 2963
}

/* Restart timer after forward progress on connection.
 * RFC2988 recommends to restart timer to now+rto.
 */
2964
void tcp_rearm_rto(struct sock *sk)
L
Linus Torvalds 已提交
2965
{
N
Nandita Dukkipati 已提交
2966
	const struct inet_connection_sock *icsk = inet_csk(sk);
2967
	struct tcp_sock *tp = tcp_sk(sk);
2968

2969 2970 2971 2972 2973 2974
	/* If the retrans timer is currently being used by Fast Open
	 * for SYN-ACK retrans purpose, stay put.
	 */
	if (tp->fastopen_rsk)
		return;

L
Linus Torvalds 已提交
2975
	if (!tp->packets_out) {
2976
		inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS);
L
Linus Torvalds 已提交
2977
	} else {
2978 2979
		u32 rto = inet_csk(sk)->icsk_rto;
		/* Offset the time elapsed after installing regular RTO */
N
Nandita Dukkipati 已提交
2980 2981
		if (icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS ||
		    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2982
			struct sk_buff *skb = tcp_write_queue_head(sk);
E
Eric Dumazet 已提交
2983 2984
			const u32 rto_time_stamp =
				tcp_skb_timestamp(skb) + rto;
2985 2986
			s32 delta = (s32)(rto_time_stamp - tcp_time_stamp);
			/* delta may not be positive if the socket is locked
N
Nandita Dukkipati 已提交
2987
			 * when the retrans timer fires and is rescheduled.
2988 2989 2990 2991 2992 2993
			 */
			if (delta > 0)
				rto = delta;
		}
		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
					  TCP_RTO_MAX);
L
Linus Torvalds 已提交
2994
	}
2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012
}

/* This function is called when the delayed ER timer fires. TCP enters
 * fast recovery and performs fast-retransmit.
 */
void tcp_resume_early_retransmit(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);

	tcp_rearm_rto(sk);

	/* Stop if ER is disabled after the delayed ER timer is scheduled */
	if (!tp->do_early_retrans)
		return;

	tcp_enter_recovery(sk, false);
	tcp_update_scoreboard(sk, 1);
	tcp_xmit_retransmit_queue(sk);
L
Linus Torvalds 已提交
3013 3014
}

3015
/* If we get here, the whole TSO packet has not been acked. */
3016
static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
3017 3018
{
	struct tcp_sock *tp = tcp_sk(sk);
3019
	u32 packets_acked;
L
Linus Torvalds 已提交
3020

3021
	BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una));
L
Linus Torvalds 已提交
3022 3023

	packets_acked = tcp_skb_pcount(skb);
3024
	if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
L
Linus Torvalds 已提交
3025 3026 3027 3028 3029
		return 0;
	packets_acked -= tcp_skb_pcount(skb);

	if (packets_acked) {
		BUG_ON(tcp_skb_pcount(skb) == 0);
3030
		BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq));
L
Linus Torvalds 已提交
3031 3032
	}

3033
	return packets_acked;
L
Linus Torvalds 已提交
3034 3035
}

3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050
static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb,
			   u32 prior_snd_una)
{
	const struct skb_shared_info *shinfo;

	/* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */
	if (likely(!(sk->sk_tsflags & SOF_TIMESTAMPING_TX_ACK)))
		return;

	shinfo = skb_shinfo(skb);
	if ((shinfo->tx_flags & SKBTX_ACK_TSTAMP) &&
	    between(shinfo->tskey, prior_snd_una, tcp_sk(sk)->snd_una - 1))
		__skb_tstamp_tx(skb, NULL, sk, SCM_TSTAMP_ACK);
}

3051 3052 3053 3054
/* Remove acknowledged frames from the retransmission queue. If our packet
 * is before the ack sequence we can discard it as it's confirmed to have
 * arrived at the other end.
 */
I
Ilpo Järvinen 已提交
3055
static int tcp_clean_rtx_queue(struct sock *sk, int prior_fackets,
3056
			       u32 prior_snd_una, long sack_rtt_us)
L
Linus Torvalds 已提交
3057
{
3058
	const struct inet_connection_sock *icsk = inet_csk(sk);
3059 3060 3061 3062
	struct skb_mstamp first_ackt, last_ackt, now;
	struct tcp_sock *tp = tcp_sk(sk);
	u32 prior_sacked = tp->sacked_out;
	u32 reord = tp->packets_out;
3063
	bool fully_acked = true;
3064 3065 3066
	long ca_seq_rtt_us = -1L;
	long seq_rtt_us = -1L;
	struct sk_buff *skb;
3067
	u32 pkts_acked = 0;
3068
	bool rtt_update;
3069 3070 3071
	int flag = 0;

	first_ackt.v64 = 0;
L
Linus Torvalds 已提交
3072

3073
	while ((skb = tcp_write_queue_head(sk)) && skb != tcp_send_head(sk)) {
3074
		struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
3075
		u8 sacked = scb->sacked;
3076
		u32 acked_pcount;
L
Linus Torvalds 已提交
3077

3078
		tcp_ack_tstamp(sk, skb, prior_snd_una);
3079

3080
		/* Determine how many packets and what bytes were acked, tso and else */
L
Linus Torvalds 已提交
3081
		if (after(scb->end_seq, tp->snd_una)) {
3082 3083 3084 3085
			if (tcp_skb_pcount(skb) == 1 ||
			    !after(tp->snd_una, scb->seq))
				break;

3086 3087
			acked_pcount = tcp_tso_acked(sk, skb);
			if (!acked_pcount)
3088 3089
				break;

E
Eric Dumazet 已提交
3090
			fully_acked = false;
3091
		} else {
3092 3093
			/* Speedup tcp_unlink_write_queue() and next loop */
			prefetchw(skb->next);
3094
			acked_pcount = tcp_skb_pcount(skb);
L
Linus Torvalds 已提交
3095 3096
		}

3097
		if (unlikely(sacked & TCPCB_RETRANS)) {
3098
			if (sacked & TCPCB_SACKED_RETRANS)
3099
				tp->retrans_out -= acked_pcount;
3100
			flag |= FLAG_RETRANS_DATA_ACKED;
3101
		} else {
3102
			last_ackt = skb->skb_mstamp;
E
Eric Dumazet 已提交
3103
			WARN_ON_ONCE(last_ackt.v64 == 0);
3104 3105 3106
			if (!first_ackt.v64)
				first_ackt = last_ackt;

3107
			if (!(sacked & TCPCB_SACKED_ACKED))
3108
				reord = min(pkts_acked, reord);
Y
Yuchung Cheng 已提交
3109 3110
			if (!after(scb->end_seq, tp->high_seq))
				flag |= FLAG_ORIG_SACK_ACKED;
3111
		}
3112 3113

		if (sacked & TCPCB_SACKED_ACKED)
3114
			tp->sacked_out -= acked_pcount;
3115
		if (sacked & TCPCB_LOST)
3116
			tp->lost_out -= acked_pcount;
3117

3118 3119
		tp->packets_out -= acked_pcount;
		pkts_acked += acked_pcount;
3120

3121 3122 3123 3124 3125 3126 3127
		/* Initial outgoing SYN's get put onto the write_queue
		 * just like anything else we transmit.  It is not
		 * true data, and if we misinform our callers that
		 * this ACK acks real data, we will erroneously exit
		 * connection startup slow start one packet too
		 * quickly.  This is severely frowned upon behavior.
		 */
3128
		if (likely(!(scb->tcp_flags & TCPHDR_SYN))) {
3129 3130 3131 3132 3133 3134
			flag |= FLAG_DATA_ACKED;
		} else {
			flag |= FLAG_SYN_ACKED;
			tp->retrans_stamp = 0;
		}

3135 3136 3137
		if (!fully_acked)
			break;

3138
		tcp_unlink_write_queue(skb, sk);
3139
		sk_wmem_free_skb(sk, skb);
3140
		if (unlikely(skb == tp->retransmit_skb_hint))
3141
			tp->retransmit_skb_hint = NULL;
3142
		if (unlikely(skb == tp->lost_skb_hint))
3143
			tp->lost_skb_hint = NULL;
L
Linus Torvalds 已提交
3144 3145
	}

I
Ilpo Järvinen 已提交
3146 3147 3148
	if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una)))
		tp->snd_up = tp->snd_una;

3149 3150 3151
	if (skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
		flag |= FLAG_SACK_RENEGING;

3152
	skb_mstamp_get(&now);
3153
	if (likely(first_ackt.v64)) {
3154 3155 3156 3157 3158
		seq_rtt_us = skb_mstamp_us_delta(&now, &first_ackt);
		ca_seq_rtt_us = skb_mstamp_us_delta(&now, &last_ackt);
	}

	rtt_update = tcp_ack_update_rtt(sk, flag, seq_rtt_us, sack_rtt_us);
Y
Yuchung Cheng 已提交
3159

3160
	if (flag & FLAG_ACKED) {
3161 3162 3163
		const struct tcp_congestion_ops *ca_ops
			= inet_csk(sk)->icsk_ca_ops;

3164
		tcp_rearm_rto(sk);
3165 3166 3167 3168 3169
		if (unlikely(icsk->icsk_mtup.probe_size &&
			     !after(tp->mtu_probe.probe_seq_end, tp->snd_una))) {
			tcp_mtup_probe_success(sk);
		}

3170 3171 3172
		if (tcp_is_reno(tp)) {
			tcp_remove_reno_sacks(sk, pkts_acked);
		} else {
3173 3174
			int delta;

3175 3176 3177
			/* Non-retransmitted hole got filled? That's reordering */
			if (reord < prior_fackets)
				tcp_update_reordering(sk, tp->fackets_out - reord, 0);
3178

3179 3180 3181
			delta = tcp_is_fack(tp) ? pkts_acked :
						  prior_sacked - tp->sacked_out;
			tp->lost_cnt_hint -= min(tp->lost_cnt_hint, delta);
3182 3183
		}

I
Ilpo Järvinen 已提交
3184
		tp->fackets_out -= min(pkts_acked, tp->fackets_out);
3185

3186 3187
		if (ca_ops->pkts_acked)
			ca_ops->pkts_acked(sk, pkts_acked, ca_seq_rtt_us);
3188

3189 3190
	} else if (skb && rtt_update && sack_rtt_us >= 0 &&
		   sack_rtt_us > skb_mstamp_us_delta(&now, &skb->skb_mstamp)) {
3191 3192 3193 3194 3195
		/* Do not re-arm RTO if the sack RTT is measured from data sent
		 * after when the head was last (re)transmitted. Otherwise the
		 * timeout may continue to extend in loss recovery.
		 */
		tcp_rearm_rto(sk);
L
Linus Torvalds 已提交
3196 3197 3198
	}

#if FASTRETRANS_DEBUG > 0
3199 3200 3201
	WARN_ON((int)tp->sacked_out < 0);
	WARN_ON((int)tp->lost_out < 0);
	WARN_ON((int)tp->retrans_out < 0);
3202
	if (!tp->packets_out && tcp_is_sack(tp)) {
S
Stephen Hemminger 已提交
3203
		icsk = inet_csk(sk);
L
Linus Torvalds 已提交
3204
		if (tp->lost_out) {
3205 3206
			pr_debug("Leak l=%u %d\n",
				 tp->lost_out, icsk->icsk_ca_state);
L
Linus Torvalds 已提交
3207 3208 3209
			tp->lost_out = 0;
		}
		if (tp->sacked_out) {
3210 3211
			pr_debug("Leak s=%u %d\n",
				 tp->sacked_out, icsk->icsk_ca_state);
L
Linus Torvalds 已提交
3212 3213 3214
			tp->sacked_out = 0;
		}
		if (tp->retrans_out) {
3215 3216
			pr_debug("Leak r=%u %d\n",
				 tp->retrans_out, icsk->icsk_ca_state);
L
Linus Torvalds 已提交
3217 3218 3219 3220
			tp->retrans_out = 0;
		}
	}
#endif
3221
	return flag;
L
Linus Torvalds 已提交
3222 3223 3224 3225
}

static void tcp_ack_probe(struct sock *sk)
{
3226 3227
	const struct tcp_sock *tp = tcp_sk(sk);
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
3228 3229 3230

	/* Was it a usable window open? */

3231
	if (!after(TCP_SKB_CB(tcp_send_head(sk))->end_seq, tcp_wnd_end(tp))) {
3232 3233
		icsk->icsk_backoff = 0;
		inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0);
L
Linus Torvalds 已提交
3234 3235 3236 3237
		/* Socket must be waked up by subsequent tcp_data_snd_check().
		 * This function is not for random using!
		 */
	} else {
3238 3239
		unsigned long when = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);

3240
		inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3241
					  when, TCP_RTO_MAX);
L
Linus Torvalds 已提交
3242 3243 3244
	}
}

3245
static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag)
L
Linus Torvalds 已提交
3246
{
E
Eric Dumazet 已提交
3247 3248
	return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) ||
		inet_csk(sk)->icsk_ca_state != TCP_CA_Open;
L
Linus Torvalds 已提交
3249 3250
}

3251
/* Decide wheather to run the increase function of congestion control. */
3252
static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
L
Linus Torvalds 已提交
3253
{
3254 3255 3256 3257 3258
	if (tcp_in_cwnd_reduction(sk))
		return false;

	/* If reordering is high then always grow cwnd whenever data is
	 * delivered regardless of its ordering. Otherwise stay conservative
3259
	 * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/
3260 3261 3262 3263 3264 3265
	 * new SACK or ECE mark may first advance cwnd here and later reduce
	 * cwnd in tcp_fastretrans_alert() based on more states.
	 */
	if (tcp_sk(sk)->reordering > sysctl_tcp_reordering)
		return flag & FLAG_FORWARD_PROGRESS;

3266
	return flag & FLAG_DATA_ACKED;
L
Linus Torvalds 已提交
3267 3268 3269 3270 3271
}

/* Check that window update is acceptable.
 * The function assumes that snd_una<=ack<=snd_next.
 */
3272
static inline bool tcp_may_update_window(const struct tcp_sock *tp,
3273 3274
					const u32 ack, const u32 ack_seq,
					const u32 nwin)
L
Linus Torvalds 已提交
3275
{
E
Eric Dumazet 已提交
3276
	return	after(ack, tp->snd_una) ||
L
Linus Torvalds 已提交
3277
		after(ack_seq, tp->snd_wl1) ||
E
Eric Dumazet 已提交
3278
		(ack_seq == tp->snd_wl1 && nwin > tp->snd_wnd);
L
Linus Torvalds 已提交
3279 3280 3281 3282 3283 3284 3285
}

/* Update our send window.
 *
 * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2
 * and in FreeBSD. NetBSD's one is even worse.) is wrong.
 */
3286
static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack,
3287
				 u32 ack_seq)
L
Linus Torvalds 已提交
3288
{
3289
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
3290
	int flag = 0;
3291
	u32 nwin = ntohs(tcp_hdr(skb)->window);
L
Linus Torvalds 已提交
3292

3293
	if (likely(!tcp_hdr(skb)->syn))
L
Linus Torvalds 已提交
3294 3295 3296 3297
		nwin <<= tp->rx_opt.snd_wscale;

	if (tcp_may_update_window(tp, ack, ack_seq, nwin)) {
		flag |= FLAG_WIN_UPDATE;
3298
		tcp_update_wl(tp, ack_seq);
L
Linus Torvalds 已提交
3299 3300 3301 3302 3303 3304 3305

		if (tp->snd_wnd != nwin) {
			tp->snd_wnd = nwin;

			/* Note, it is the only place, where
			 * fast path is recovered for sending TCP.
			 */
3306
			tp->pred_flags = 0;
3307
			tcp_fast_path_check(sk);
L
Linus Torvalds 已提交
3308 3309 3310

			if (nwin > tp->max_window) {
				tp->max_window = nwin;
3311
				tcp_sync_mss(sk, inet_csk(sk)->icsk_pmtu_cookie);
L
Linus Torvalds 已提交
3312 3313 3314 3315 3316 3317 3318 3319 3320
			}
		}
	}

	tp->snd_una = ack;

	return flag;
}

3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338
/* RFC 5961 7 [ACK Throttling] */
static void tcp_send_challenge_ack(struct sock *sk)
{
	/* unprotected vars, we dont care of overwrites */
	static u32 challenge_timestamp;
	static unsigned int challenge_count;
	u32 now = jiffies / HZ;

	if (now != challenge_timestamp) {
		challenge_timestamp = now;
		challenge_count = 0;
	}
	if (++challenge_count <= sysctl_tcp_challenge_ack_limit) {
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPCHALLENGEACK);
		tcp_send_ack(sk);
	}
}

3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359
static void tcp_store_ts_recent(struct tcp_sock *tp)
{
	tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval;
	tp->rx_opt.ts_recent_stamp = get_seconds();
}

static void tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq)
{
	if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) {
		/* PAWS bug workaround wrt. ACK frames, the PAWS discard
		 * extra check below makes sure this can only happen
		 * for pure ACK frames.  -DaveM
		 *
		 * Not only, also it occurs for expired timestamps.
		 */

		if (tcp_paws_check(&tp->rx_opt, 0))
			tcp_store_ts_recent(tp);
	}
}

N
Nandita Dukkipati 已提交
3360
/* This routine deals with acks during a TLP episode.
3361 3362
 * We mark the end of a TLP episode on receiving TLP dupack or when
 * ack is after tlp_high_seq.
N
Nandita Dukkipati 已提交
3363 3364 3365 3366 3367 3368
 * Ref: loss detection algorithm in draft-dukkipati-tcpm-tcp-loss-probe.
 */
static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag)
{
	struct tcp_sock *tp = tcp_sk(sk);

3369
	if (before(ack, tp->tlp_high_seq))
N
Nandita Dukkipati 已提交
3370 3371
		return;

3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387
	if (flag & FLAG_DSACKING_ACK) {
		/* This DSACK means original and TLP probe arrived; no loss */
		tp->tlp_high_seq = 0;
	} else if (after(ack, tp->tlp_high_seq)) {
		/* ACK advances: there was a loss, so reduce cwnd. Reset
		 * tlp_high_seq in tcp_init_cwnd_reduction()
		 */
		tcp_init_cwnd_reduction(sk);
		tcp_set_ca_state(sk, TCP_CA_CWR);
		tcp_end_cwnd_reduction(sk);
		tcp_try_keep_open(sk);
		NET_INC_STATS_BH(sock_net(sk),
				 LINUX_MIB_TCPLOSSPROBERECOVERY);
	} else if (!(flag & (FLAG_SND_UNA_ADVANCED |
			     FLAG_NOT_DUP | FLAG_DATA_SACKED))) {
		/* Pure dupack: original and TLP probe arrived; no loss */
N
Nandita Dukkipati 已提交
3388 3389 3390 3391
		tp->tlp_high_seq = 0;
	}
}

3392 3393 3394 3395 3396 3397 3398 3399
static inline void tcp_in_ack_event(struct sock *sk, u32 flags)
{
	const struct inet_connection_sock *icsk = inet_csk(sk);

	if (icsk->icsk_ca_ops->in_ack_event)
		icsk->icsk_ca_ops->in_ack_event(sk, flags);
}

L
Linus Torvalds 已提交
3400
/* This routine deals with incoming acks, but not outgoing ones. */
3401
static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
L
Linus Torvalds 已提交
3402
{
3403
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
3404 3405 3406 3407
	struct tcp_sock *tp = tcp_sk(sk);
	u32 prior_snd_una = tp->snd_una;
	u32 ack_seq = TCP_SKB_CB(skb)->seq;
	u32 ack = TCP_SKB_CB(skb)->ack_seq;
3408
	bool is_dupack = false;
3409
	u32 prior_fackets;
3410
	int prior_packets = tp->packets_out;
3411 3412
	const int prior_unsacked = tp->packets_out - tp->sacked_out;
	int acked = 0; /* Number of packets newly acked */
3413
	long sack_rtt_us = -1L;
L
Linus Torvalds 已提交
3414

3415 3416 3417
	/* We very likely will need to access write queue head. */
	prefetchw(sk->sk_write_queue.next);

3418
	/* If the ack is older than previous acks
L
Linus Torvalds 已提交
3419 3420
	 * then we can probably ignore it.
	 */
3421 3422 3423 3424 3425 3426
	if (before(ack, prior_snd_una)) {
		/* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */
		if (before(ack, prior_snd_una - tp->max_window)) {
			tcp_send_challenge_ack(sk);
			return -1;
		}
L
Linus Torvalds 已提交
3427
		goto old_ack;
3428
	}
L
Linus Torvalds 已提交
3429

3430 3431 3432 3433 3434 3435
	/* If the ack includes data we haven't sent yet, discard
	 * this segment (RFC793 Section 3.9).
	 */
	if (after(ack, tp->snd_nxt))
		goto invalid_ack;

N
Nandita Dukkipati 已提交
3436 3437
	if (icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS ||
	    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
3438 3439
		tcp_rearm_rto(sk);

3440
	if (after(ack, prior_snd_una)) {
3441
		flag |= FLAG_SND_UNA_ADVANCED;
3442 3443
		icsk->icsk_retransmits = 0;
	}
3444

3445 3446
	prior_fackets = tp->fackets_out;

3447 3448 3449 3450 3451 3452
	/* ts_recent update must be made after we are sure that the packet
	 * is in window.
	 */
	if (flag & FLAG_UPDATE_TS_RECENT)
		tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);

3453
	if (!(flag & FLAG_SLOWPATH) && after(ack, prior_snd_una)) {
L
Linus Torvalds 已提交
3454 3455 3456 3457
		/* Window is constant, pure forward advance.
		 * No more checks are required.
		 * Note, we use the fact that SND.UNA>=SND.WL2.
		 */
3458
		tcp_update_wl(tp, ack_seq);
L
Linus Torvalds 已提交
3459 3460 3461
		tp->snd_una = ack;
		flag |= FLAG_WIN_UPDATE;

3462
		tcp_in_ack_event(sk, CA_ACK_WIN_UPDATE);
3463

3464
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPACKS);
L
Linus Torvalds 已提交
3465
	} else {
3466 3467
		u32 ack_ev_flags = CA_ACK_SLOWPATH;

L
Linus Torvalds 已提交
3468 3469 3470
		if (ack_seq != TCP_SKB_CB(skb)->end_seq)
			flag |= FLAG_DATA;
		else
3471
			NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPPUREACKS);
L
Linus Torvalds 已提交
3472

3473
		flag |= tcp_ack_update_window(sk, skb, ack, ack_seq);
L
Linus Torvalds 已提交
3474 3475

		if (TCP_SKB_CB(skb)->sacked)
Y
Yuchung Cheng 已提交
3476
			flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
3477
							&sack_rtt_us);
L
Linus Torvalds 已提交
3478

3479
		if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb))) {
L
Linus Torvalds 已提交
3480
			flag |= FLAG_ECE;
3481 3482 3483 3484 3485
			ack_ev_flags |= CA_ACK_ECE;
		}

		if (flag & FLAG_WIN_UPDATE)
			ack_ev_flags |= CA_ACK_WIN_UPDATE;
L
Linus Torvalds 已提交
3486

3487
		tcp_in_ack_event(sk, ack_ev_flags);
L
Linus Torvalds 已提交
3488 3489 3490 3491 3492 3493
	}

	/* We passed data and got it acked, remove any soft error
	 * log. Something worked...
	 */
	sk->sk_err_soft = 0;
3494
	icsk->icsk_probes_out = 0;
L
Linus Torvalds 已提交
3495 3496 3497 3498 3499
	tp->rcv_tstamp = tcp_time_stamp;
	if (!prior_packets)
		goto no_queue;

	/* See if we can take anything off of the retransmit queue. */
3500
	acked = tp->packets_out;
3501 3502
	flag |= tcp_clean_rtx_queue(sk, prior_fackets, prior_snd_una,
				    sack_rtt_us);
3503
	acked -= tp->packets_out;
3504

3505 3506
	/* Advance cwnd if state allows */
	if (tcp_may_raise_cwnd(sk, flag))
3507
		tcp_cong_avoid(sk, ack, acked);
3508

3509
	if (tcp_ack_is_dubious(sk, flag)) {
3510
		is_dupack = !(flag & (FLAG_SND_UNA_ADVANCED | FLAG_NOT_DUP));
3511 3512
		tcp_fastretrans_alert(sk, acked, prior_unsacked,
				      is_dupack, flag);
L
Linus Torvalds 已提交
3513
	}
N
Nandita Dukkipati 已提交
3514 3515 3516
	if (tp->tlp_high_seq)
		tcp_process_tlp_ack(sk, ack, flag);

3517 3518 3519 3520 3521
	if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP)) {
		struct dst_entry *dst = __sk_dst_get(sk);
		if (dst)
			dst_confirm(dst);
	}
N
Nandita Dukkipati 已提交
3522 3523 3524

	if (icsk->icsk_pending == ICSK_TIME_RETRANS)
		tcp_schedule_loss_probe(sk);
3525
	tcp_update_pacing_rate(sk);
L
Linus Torvalds 已提交
3526 3527 3528
	return 1;

no_queue:
3529 3530
	/* If data was DSACKed, see if we can undo a cwnd reduction. */
	if (flag & FLAG_DSACKING_ACK)
3531 3532
		tcp_fastretrans_alert(sk, acked, prior_unsacked,
				      is_dupack, flag);
L
Linus Torvalds 已提交
3533 3534 3535 3536
	/* If this ack opens up a zero window, clear backoff.  It was
	 * being used to time the probes, and is probably far higher than
	 * it needs to be for normal retransmission.
	 */
3537
	if (tcp_send_head(sk))
L
Linus Torvalds 已提交
3538
		tcp_ack_probe(sk);
N
Nandita Dukkipati 已提交
3539 3540 3541

	if (tp->tlp_high_seq)
		tcp_process_tlp_ack(sk, ack, flag);
L
Linus Torvalds 已提交
3542 3543
	return 1;

3544 3545 3546 3547
invalid_ack:
	SOCK_DEBUG(sk, "Ack %u after %u:%u\n", ack, tp->snd_una, tp->snd_nxt);
	return -1;

L
Linus Torvalds 已提交
3548
old_ack:
3549 3550 3551
	/* If data was SACKed, tag it and see if we should send more data.
	 * If data was DSACKed, see if we can undo a cwnd reduction.
	 */
3552
	if (TCP_SKB_CB(skb)->sacked) {
Y
Yuchung Cheng 已提交
3553
		flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
3554
						&sack_rtt_us);
3555 3556
		tcp_fastretrans_alert(sk, acked, prior_unsacked,
				      is_dupack, flag);
3557
	}
L
Linus Torvalds 已提交
3558

3559
	SOCK_DEBUG(sk, "Ack %u before %u:%u\n", ack, tp->snd_una, tp->snd_nxt);
L
Linus Torvalds 已提交
3560 3561 3562 3563 3564 3565 3566
	return 0;
}

/* Look for tcp options. Normally only called on SYN and SYNACK packets.
 * But, this can also be called on packets in the established flow when
 * the fast version below fails.
 */
C
Christoph Paasch 已提交
3567 3568
void tcp_parse_options(const struct sk_buff *skb,
		       struct tcp_options_received *opt_rx, int estab,
Y
Yuchung Cheng 已提交
3569
		       struct tcp_fastopen_cookie *foc)
L
Linus Torvalds 已提交
3570
{
3571 3572
	const unsigned char *ptr;
	const struct tcphdr *th = tcp_hdr(skb);
3573
	int length = (th->doff * 4) - sizeof(struct tcphdr);
L
Linus Torvalds 已提交
3574

3575
	ptr = (const unsigned char *)(th + 1);
L
Linus Torvalds 已提交
3576 3577
	opt_rx->saw_tstamp = 0;

S
Stephen Hemminger 已提交
3578
	while (length > 0) {
3579
		int opcode = *ptr++;
L
Linus Torvalds 已提交
3580 3581 3582
		int opsize;

		switch (opcode) {
3583 3584 3585 3586 3587 3588 3589 3590
		case TCPOPT_EOL:
			return;
		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
			length--;
			continue;
		default:
			opsize = *ptr++;
			if (opsize < 2) /* "silly options" */
L
Linus Torvalds 已提交
3591
				return;
3592 3593 3594 3595 3596
			if (opsize > length)
				return;	/* don't parse partial options */
			switch (opcode) {
			case TCPOPT_MSS:
				if (opsize == TCPOLEN_MSS && th->syn && !estab) {
3597
					u16 in_mss = get_unaligned_be16(ptr);
3598 3599 3600 3601 3602
					if (in_mss) {
						if (opt_rx->user_mss &&
						    opt_rx->user_mss < in_mss)
							in_mss = opt_rx->user_mss;
						opt_rx->mss_clamp = in_mss;
L
Linus Torvalds 已提交
3603
					}
3604 3605 3606 3607
				}
				break;
			case TCPOPT_WINDOW:
				if (opsize == TCPOLEN_WINDOW && th->syn &&
3608
				    !estab && sysctl_tcp_window_scaling) {
3609 3610 3611
					__u8 snd_wscale = *(__u8 *)ptr;
					opt_rx->wscale_ok = 1;
					if (snd_wscale > 14) {
3612 3613 3614
						net_info_ratelimited("%s: Illegal window scaling value %d >14 received\n",
								     __func__,
								     snd_wscale);
3615
						snd_wscale = 14;
L
Linus Torvalds 已提交
3616
					}
3617 3618 3619 3620 3621 3622
					opt_rx->snd_wscale = snd_wscale;
				}
				break;
			case TCPOPT_TIMESTAMP:
				if ((opsize == TCPOLEN_TIMESTAMP) &&
				    ((estab && opt_rx->tstamp_ok) ||
3623
				     (!estab && sysctl_tcp_timestamps))) {
3624
					opt_rx->saw_tstamp = 1;
3625 3626
					opt_rx->rcv_tsval = get_unaligned_be32(ptr);
					opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4);
3627 3628 3629 3630
				}
				break;
			case TCPOPT_SACK_PERM:
				if (opsize == TCPOLEN_SACK_PERM && th->syn &&
3631
				    !estab && sysctl_tcp_sack) {
3632
					opt_rx->sack_ok = TCP_SACK_SEEN;
3633 3634 3635
					tcp_sack_reset(opt_rx);
				}
				break;
L
Linus Torvalds 已提交
3636

3637 3638 3639 3640 3641 3642 3643
			case TCPOPT_SACK:
				if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) &&
				   !((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) &&
				   opt_rx->sack_ok) {
					TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th;
				}
				break;
3644
#ifdef CONFIG_TCP_MD5SIG
3645 3646 3647 3648 3649 3650
			case TCPOPT_MD5SIG:
				/*
				 * The MD5 Hash has already been
				 * checked (see tcp_v{4,6}_do_rcv()).
				 */
				break;
3651
#endif
Y
Yuchung Cheng 已提交
3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669
			case TCPOPT_EXP:
				/* Fast Open option shares code 254 using a
				 * 16 bits magic number. It's valid only in
				 * SYN or SYN-ACK with an even size.
				 */
				if (opsize < TCPOLEN_EXP_FASTOPEN_BASE ||
				    get_unaligned_be16(ptr) != TCPOPT_FASTOPEN_MAGIC ||
				    foc == NULL || !th->syn || (opsize & 1))
					break;
				foc->len = opsize - TCPOLEN_EXP_FASTOPEN_BASE;
				if (foc->len >= TCP_FASTOPEN_COOKIE_MIN &&
				    foc->len <= TCP_FASTOPEN_COOKIE_MAX)
					memcpy(foc->val, ptr + 2, foc->len);
				else if (foc->len != 0)
					foc->len = -1;
				break;

			}
3670 3671
			ptr += opsize-2;
			length -= opsize;
3672
		}
L
Linus Torvalds 已提交
3673 3674
	}
}
E
Eric Dumazet 已提交
3675
EXPORT_SYMBOL(tcp_parse_options);
L
Linus Torvalds 已提交
3676

E
Eric Dumazet 已提交
3677
static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th)
3678
{
3679
	const __be32 *ptr = (const __be32 *)(th + 1);
3680 3681 3682 3683 3684 3685 3686

	if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
			  | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
		tp->rx_opt.saw_tstamp = 1;
		++ptr;
		tp->rx_opt.rcv_tsval = ntohl(*ptr);
		++ptr;
3687 3688 3689 3690
		if (*ptr)
			tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset;
		else
			tp->rx_opt.rcv_tsecr = 0;
E
Eric Dumazet 已提交
3691
		return true;
3692
	}
E
Eric Dumazet 已提交
3693
	return false;
3694 3695
}

L
Linus Torvalds 已提交
3696 3697 3698
/* Fast parse options. This hopes to only see timestamps.
 * If it is wrong it falls back on tcp_parse_options().
 */
E
Eric Dumazet 已提交
3699
static bool tcp_fast_parse_options(const struct sk_buff *skb,
C
Christoph Paasch 已提交
3700
				   const struct tcphdr *th, struct tcp_sock *tp)
L
Linus Torvalds 已提交
3701
{
3702 3703 3704 3705
	/* In the spirit of fast parsing, compare doff directly to constant
	 * values.  Because equality is used, short doff can be ignored here.
	 */
	if (th->doff == (sizeof(*th) / 4)) {
L
Linus Torvalds 已提交
3706
		tp->rx_opt.saw_tstamp = 0;
E
Eric Dumazet 已提交
3707
		return false;
L
Linus Torvalds 已提交
3708
	} else if (tp->rx_opt.tstamp_ok &&
3709
		   th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) {
3710
		if (tcp_parse_aligned_timestamp(tp, th))
E
Eric Dumazet 已提交
3711
			return true;
L
Linus Torvalds 已提交
3712
	}
3713

C
Christoph Paasch 已提交
3714
	tcp_parse_options(skb, &tp->rx_opt, 1, NULL);
3715
	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
3716 3717
		tp->rx_opt.rcv_tsecr -= tp->tsoffset;

E
Eric Dumazet 已提交
3718
	return true;
L
Linus Torvalds 已提交
3719 3720
}

3721 3722 3723 3724
#ifdef CONFIG_TCP_MD5SIG
/*
 * Parse MD5 Signature option
 */
3725
const u8 *tcp_parse_md5sig_option(const struct tcphdr *th)
3726
{
3727 3728
	int length = (th->doff << 2) - sizeof(*th);
	const u8 *ptr = (const u8 *)(th + 1);
3729 3730 3731 3732 3733 3734 3735 3736 3737

	/* If the TCP option is too short, we can short cut */
	if (length < TCPOLEN_MD5SIG)
		return NULL;

	while (length > 0) {
		int opcode = *ptr++;
		int opsize;

W
Weilong Chen 已提交
3738
		switch (opcode) {
3739 3740 3741 3742 3743 3744 3745 3746 3747 3748
		case TCPOPT_EOL:
			return NULL;
		case TCPOPT_NOP:
			length--;
			continue;
		default:
			opsize = *ptr++;
			if (opsize < 2 || opsize > length)
				return NULL;
			if (opcode == TCPOPT_MD5SIG)
3749
				return opsize == TCPOLEN_MD5SIG ? ptr : NULL;
3750 3751 3752 3753 3754 3755
		}
		ptr += opsize - 2;
		length -= opsize;
	}
	return NULL;
}
E
Eric Dumazet 已提交
3756
EXPORT_SYMBOL(tcp_parse_md5sig_option);
3757 3758
#endif

L
Linus Torvalds 已提交
3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781
/* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM
 *
 * It is not fatal. If this ACK does _not_ change critical state (seqs, window)
 * it can pass through stack. So, the following predicate verifies that
 * this segment is not used for anything but congestion avoidance or
 * fast retransmit. Moreover, we even are able to eliminate most of such
 * second order effects, if we apply some small "replay" window (~RTO)
 * to timestamp space.
 *
 * All these measures still do not guarantee that we reject wrapped ACKs
 * on networks with high bandwidth, when sequence space is recycled fastly,
 * but it guarantees that such events will be very rare and do not affect
 * connection seriously. This doesn't look nice, but alas, PAWS is really
 * buggy extension.
 *
 * [ Later note. Even worse! It is buggy for segments _with_ data. RFC
 * states that events when retransmit arrives after original data are rare.
 * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is
 * the biggest problem on large power networks even with minor reordering.
 * OK, let's give it small replay window. If peer clock is even 1hz, it is safe
 * up to bandwidth of 18Gigabit/sec. 8) ]
 */

3782
static int tcp_disordered_ack(const struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
3783
{
3784 3785
	const struct tcp_sock *tp = tcp_sk(sk);
	const struct tcphdr *th = tcp_hdr(skb);
L
Linus Torvalds 已提交
3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798
	u32 seq = TCP_SKB_CB(skb)->seq;
	u32 ack = TCP_SKB_CB(skb)->ack_seq;

	return (/* 1. Pure ACK with correct sequence number. */
		(th->ack && seq == TCP_SKB_CB(skb)->end_seq && seq == tp->rcv_nxt) &&

		/* 2. ... and duplicate ACK. */
		ack == tp->snd_una &&

		/* 3. ... and does not update window. */
		!tcp_may_update_window(tp, ack, seq, ntohs(th->window) << tp->rx_opt.snd_wscale) &&

		/* 4. ... and sits in replay window. */
3799
		(s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) <= (inet_csk(sk)->icsk_rto * 1024) / HZ);
L
Linus Torvalds 已提交
3800 3801
}

3802
static inline bool tcp_paws_discard(const struct sock *sk,
3803
				   const struct sk_buff *skb)
L
Linus Torvalds 已提交
3804
{
3805
	const struct tcp_sock *tp = tcp_sk(sk);
I
Ilpo Järvinen 已提交
3806 3807 3808

	return !tcp_paws_check(&tp->rx_opt, TCP_PAWS_WINDOW) &&
	       !tcp_disordered_ack(sk, skb);
L
Linus Torvalds 已提交
3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823
}

/* Check segment sequence number for validity.
 *
 * Segment controls are considered valid, if the segment
 * fits to the window after truncation to the window. Acceptability
 * of data (and SYN, FIN, of course) is checked separately.
 * See tcp_data_queue(), for example.
 *
 * Also, controls (RST is main one) are accepted using RCV.WUP instead
 * of RCV.NXT. Peer still did not advance his SND.UNA when we
 * delayed ACK, so that hisSND.UNA<=ourRCV.WUP.
 * (borrowed from freebsd)
 */

3824
static inline bool tcp_sequence(const struct tcp_sock *tp, u32 seq, u32 end_seq)
L
Linus Torvalds 已提交
3825 3826 3827 3828 3829 3830
{
	return	!before(end_seq, tp->rcv_wup) &&
		!after(seq, tp->rcv_nxt + tcp_receive_window(tp));
}

/* When we get a reset we do this. */
3831
void tcp_reset(struct sock *sk)
L
Linus Torvalds 已提交
3832 3833 3834
{
	/* We want the right error as BSD sees it (and indeed as we do). */
	switch (sk->sk_state) {
3835 3836 3837 3838 3839 3840 3841 3842 3843 3844
	case TCP_SYN_SENT:
		sk->sk_err = ECONNREFUSED;
		break;
	case TCP_CLOSE_WAIT:
		sk->sk_err = EPIPE;
		break;
	case TCP_CLOSE:
		return;
	default:
		sk->sk_err = ECONNRESET;
L
Linus Torvalds 已提交
3845
	}
T
Tom Marshall 已提交
3846 3847
	/* This barrier is coupled with smp_rmb() in tcp_poll() */
	smp_wmb();
L
Linus Torvalds 已提交
3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868

	if (!sock_flag(sk, SOCK_DEAD))
		sk->sk_error_report(sk);

	tcp_done(sk);
}

/*
 * 	Process the FIN bit. This now behaves as it is supposed to work
 *	and the FIN takes effect when it is validly part of sequence
 *	space. Not before when we get holes.
 *
 *	If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT
 *	(and thence onto LAST-ACK and finally, CLOSE, we never enter
 *	TIME-WAIT)
 *
 *	If we are in FINWAIT-1, a received FIN indicates simultaneous
 *	close and we go into CLOSING (and later onto TIME-WAIT)
 *
 *	If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT.
 */
3869
static void tcp_fin(struct sock *sk)
L
Linus Torvalds 已提交
3870 3871
{
	struct tcp_sock *tp = tcp_sk(sk);
3872
	const struct dst_entry *dst;
L
Linus Torvalds 已提交
3873

3874
	inet_csk_schedule_ack(sk);
L
Linus Torvalds 已提交
3875 3876 3877 3878 3879

	sk->sk_shutdown |= RCV_SHUTDOWN;
	sock_set_flag(sk, SOCK_DONE);

	switch (sk->sk_state) {
3880 3881 3882 3883
	case TCP_SYN_RECV:
	case TCP_ESTABLISHED:
		/* Move to CLOSE_WAIT */
		tcp_set_state(sk, TCP_CLOSE_WAIT);
3884 3885 3886
		dst = __sk_dst_get(sk);
		if (!dst || !dst_metric(dst, RTAX_QUICKACK))
			inet_csk(sk)->icsk_ack.pingpong = 1;
3887
		break;
L
Linus Torvalds 已提交
3888

3889 3890 3891 3892 3893 3894 3895 3896 3897
	case TCP_CLOSE_WAIT:
	case TCP_CLOSING:
		/* Received a retransmission of the FIN, do
		 * nothing.
		 */
		break;
	case TCP_LAST_ACK:
		/* RFC793: Remain in the LAST-ACK state. */
		break;
L
Linus Torvalds 已提交
3898

3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915
	case TCP_FIN_WAIT1:
		/* This case occurs when a simultaneous close
		 * happens, we must ack the received FIN and
		 * enter the CLOSING state.
		 */
		tcp_send_ack(sk);
		tcp_set_state(sk, TCP_CLOSING);
		break;
	case TCP_FIN_WAIT2:
		/* Received a FIN -- send ACK and enter TIME_WAIT. */
		tcp_send_ack(sk);
		tcp_time_wait(sk, TCP_TIME_WAIT, 0);
		break;
	default:
		/* Only TCP_LISTEN and TCP_CLOSE are left, in these
		 * cases we should never reach this piece of code.
		 */
J
Joe Perches 已提交
3916
		pr_err("%s: Impossible, sk->sk_state=%d\n",
3917
		       __func__, sk->sk_state);
3918
		break;
3919
	}
L
Linus Torvalds 已提交
3920 3921 3922 3923 3924

	/* It _is_ possible, that we have something out-of-order _after_ FIN.
	 * Probably, we should reset in this case. For now drop them.
	 */
	__skb_queue_purge(&tp->out_of_order_queue);
3925
	if (tcp_is_sack(tp))
L
Linus Torvalds 已提交
3926
		tcp_sack_reset(&tp->rx_opt);
3927
	sk_mem_reclaim(sk);
L
Linus Torvalds 已提交
3928 3929 3930 3931 3932 3933 3934

	if (!sock_flag(sk, SOCK_DEAD)) {
		sk->sk_state_change(sk);

		/* Do not send POLL_HUP for half duplex close. */
		if (sk->sk_shutdown == SHUTDOWN_MASK ||
		    sk->sk_state == TCP_CLOSE)
3935
			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
L
Linus Torvalds 已提交
3936
		else
3937
			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
L
Linus Torvalds 已提交
3938 3939 3940
	}
}

E
Eric Dumazet 已提交
3941
static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq,
3942
				  u32 end_seq)
L
Linus Torvalds 已提交
3943 3944 3945 3946 3947 3948
{
	if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) {
		if (before(seq, sp->start_seq))
			sp->start_seq = seq;
		if (after(end_seq, sp->end_seq))
			sp->end_seq = end_seq;
E
Eric Dumazet 已提交
3949
		return true;
L
Linus Torvalds 已提交
3950
	}
E
Eric Dumazet 已提交
3951
	return false;
L
Linus Torvalds 已提交
3952 3953
}

3954
static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq)
L
Linus Torvalds 已提交
3955
{
3956 3957
	struct tcp_sock *tp = tcp_sk(sk);

3958
	if (tcp_is_sack(tp) && sysctl_tcp_dsack) {
3959 3960
		int mib_idx;

L
Linus Torvalds 已提交
3961
		if (before(seq, tp->rcv_nxt))
3962
			mib_idx = LINUX_MIB_TCPDSACKOLDSENT;
L
Linus Torvalds 已提交
3963
		else
3964 3965
			mib_idx = LINUX_MIB_TCPDSACKOFOSENT;

3966
		NET_INC_STATS_BH(sock_net(sk), mib_idx);
L
Linus Torvalds 已提交
3967 3968 3969 3970 3971 3972 3973

		tp->rx_opt.dsack = 1;
		tp->duplicate_sack[0].start_seq = seq;
		tp->duplicate_sack[0].end_seq = end_seq;
	}
}

3974
static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq)
L
Linus Torvalds 已提交
3975
{
3976 3977
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
3978
	if (!tp->rx_opt.dsack)
3979
		tcp_dsack_set(sk, seq, end_seq);
L
Linus Torvalds 已提交
3980 3981 3982 3983
	else
		tcp_sack_extend(tp->duplicate_sack, seq, end_seq);
}

3984
static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
3985 3986 3987 3988 3989
{
	struct tcp_sock *tp = tcp_sk(sk);

	if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
	    before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
3990
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
3991
		tcp_enter_quickack_mode(sk);
L
Linus Torvalds 已提交
3992

3993
		if (tcp_is_sack(tp) && sysctl_tcp_dsack) {
L
Linus Torvalds 已提交
3994 3995 3996 3997
			u32 end_seq = TCP_SKB_CB(skb)->end_seq;

			if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))
				end_seq = tp->rcv_nxt;
3998
			tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq);
L
Linus Torvalds 已提交
3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011
		}
	}

	tcp_send_ack(sk);
}

/* These routines update the SACK block as out-of-order packets arrive or
 * in-order packets close up the sequence space.
 */
static void tcp_sack_maybe_coalesce(struct tcp_sock *tp)
{
	int this_sack;
	struct tcp_sack_block *sp = &tp->selective_acks[0];
4012
	struct tcp_sack_block *swalk = sp + 1;
L
Linus Torvalds 已提交
4013 4014 4015 4016

	/* See if the recent change to the first SACK eats into
	 * or hits the sequence space of other SACK blocks, if so coalesce.
	 */
4017
	for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) {
L
Linus Torvalds 已提交
4018 4019 4020 4021 4022 4023 4024
		if (tcp_sack_extend(sp, swalk->start_seq, swalk->end_seq)) {
			int i;

			/* Zap SWALK, by moving every further SACK up by one slot.
			 * Decrease num_sacks.
			 */
			tp->rx_opt.num_sacks--;
4025 4026
			for (i = this_sack; i < tp->rx_opt.num_sacks; i++)
				sp[i] = sp[i + 1];
L
Linus Torvalds 已提交
4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042
			continue;
		}
		this_sack++, swalk++;
	}
}

static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq)
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct tcp_sack_block *sp = &tp->selective_acks[0];
	int cur_sacks = tp->rx_opt.num_sacks;
	int this_sack;

	if (!cur_sacks)
		goto new_sack;

4043
	for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) {
L
Linus Torvalds 已提交
4044 4045
		if (tcp_sack_extend(sp, seq, end_seq)) {
			/* Rotate this_sack to the first one. */
4046
			for (; this_sack > 0; this_sack--, sp--)
4047
				swap(*sp, *(sp - 1));
L
Linus Torvalds 已提交
4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
			if (cur_sacks > 1)
				tcp_sack_maybe_coalesce(tp);
			return;
		}
	}

	/* Could not find an adjacent existing SACK, build a new one,
	 * put it at the front, and shift everyone else down.  We
	 * always know there is at least one SACK present already here.
	 *
	 * If the sack array is full, forget about the last one.
	 */
4060
	if (this_sack >= TCP_NUM_SACKS) {
L
Linus Torvalds 已提交
4061 4062 4063 4064
		this_sack--;
		tp->rx_opt.num_sacks--;
		sp--;
	}
S
Stephen Hemminger 已提交
4065
	for (; this_sack > 0; this_sack--, sp--)
4066
		*sp = *(sp - 1);
L
Linus Torvalds 已提交
4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083

new_sack:
	/* Build the new head SACK, and we're done. */
	sp->start_seq = seq;
	sp->end_seq = end_seq;
	tp->rx_opt.num_sacks++;
}

/* RCV.NXT advances, some SACKs should be eaten. */

static void tcp_sack_remove(struct tcp_sock *tp)
{
	struct tcp_sack_block *sp = &tp->selective_acks[0];
	int num_sacks = tp->rx_opt.num_sacks;
	int this_sack;

	/* Empty ofo queue, hence, all the SACKs are eaten. Clear. */
4084
	if (skb_queue_empty(&tp->out_of_order_queue)) {
L
Linus Torvalds 已提交
4085 4086 4087 4088
		tp->rx_opt.num_sacks = 0;
		return;
	}

4089
	for (this_sack = 0; this_sack < num_sacks;) {
L
Linus Torvalds 已提交
4090 4091 4092 4093 4094
		/* Check if the start of the sack is covered by RCV.NXT. */
		if (!before(tp->rcv_nxt, sp->start_seq)) {
			int i;

			/* RCV.NXT must cover all the block! */
4095
			WARN_ON(before(tp->rcv_nxt, sp->end_seq));
L
Linus Torvalds 已提交
4096 4097

			/* Zap this SACK, by moving forward any other SACKS. */
W
Weilong Chen 已提交
4098
			for (i = this_sack+1; i < num_sacks; i++)
L
Linus Torvalds 已提交
4099 4100 4101 4102 4103 4104 4105
				tp->selective_acks[i-1] = tp->selective_acks[i];
			num_sacks--;
			continue;
		}
		this_sack++;
		sp++;
	}
4106
	tp->rx_opt.num_sacks = num_sacks;
L
Linus Torvalds 已提交
4107 4108
}

4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146
/**
 * tcp_try_coalesce - try to merge skb to prior one
 * @sk: socket
 * @to: prior buffer
 * @from: buffer to add in queue
 * @fragstolen: pointer to boolean
 *
 * Before queueing skb @from after @to, try to merge them
 * to reduce overall memory use and queue lengths, if cost is small.
 * Packets in ofo or receive queues can stay a long time.
 * Better try to coalesce them right now to avoid future collapses.
 * Returns true if caller should free @from instead of queueing it
 */
static bool tcp_try_coalesce(struct sock *sk,
			     struct sk_buff *to,
			     struct sk_buff *from,
			     bool *fragstolen)
{
	int delta;

	*fragstolen = false;

	/* Its possible this segment overlaps with prior segment in queue */
	if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq)
		return false;

	if (!skb_try_coalesce(to, from, fragstolen, &delta))
		return false;

	atomic_add(delta, &sk->sk_rmem_alloc);
	sk_mem_charge(sk, delta);
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE);
	TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq;
	TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq;
	TCP_SKB_CB(to)->tcp_flags |= TCP_SKB_CB(from)->tcp_flags;
	return true;
}

L
Linus Torvalds 已提交
4147 4148 4149 4150 4151 4152 4153
/* This one checks to see if we can put data from the
 * out_of_order queue into the receive_queue.
 */
static void tcp_ofo_queue(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	__u32 dsack_high = tp->rcv_nxt;
4154 4155
	struct sk_buff *skb, *tail;
	bool fragstolen, eaten;
L
Linus Torvalds 已提交
4156 4157 4158 4159 4160 4161 4162 4163 4164

	while ((skb = skb_peek(&tp->out_of_order_queue)) != NULL) {
		if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
			break;

		if (before(TCP_SKB_CB(skb)->seq, dsack_high)) {
			__u32 dsack = dsack_high;
			if (before(TCP_SKB_CB(skb)->end_seq, dsack_high))
				dsack_high = TCP_SKB_CB(skb)->end_seq;
4165
			tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack);
L
Linus Torvalds 已提交
4166 4167
		}

4168
		__skb_unlink(skb, &tp->out_of_order_queue);
L
Linus Torvalds 已提交
4169
		if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
4170
			SOCK_DEBUG(sk, "ofo packet was already received\n");
L
Linus Torvalds 已提交
4171 4172 4173 4174 4175 4176 4177
			__kfree_skb(skb);
			continue;
		}
		SOCK_DEBUG(sk, "ofo requeuing : rcv_next %X seq %X - %X\n",
			   tp->rcv_nxt, TCP_SKB_CB(skb)->seq,
			   TCP_SKB_CB(skb)->end_seq);

4178 4179
		tail = skb_peek_tail(&sk->sk_receive_queue);
		eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen);
L
Linus Torvalds 已提交
4180
		tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq;
4181 4182
		if (!eaten)
			__skb_queue_tail(&sk->sk_receive_queue, skb);
4183
		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
4184
			tcp_fin(sk);
4185 4186
		if (eaten)
			kfree_skb_partial(skb, fragstolen);
L
Linus Torvalds 已提交
4187 4188 4189
	}
}

E
Eric Dumazet 已提交
4190
static bool tcp_prune_ofo_queue(struct sock *sk);
L
Linus Torvalds 已提交
4191 4192
static int tcp_prune_queue(struct sock *sk);

4193 4194
static int tcp_try_rmem_schedule(struct sock *sk, struct sk_buff *skb,
				 unsigned int size)
4195 4196
{
	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
4197
	    !sk_rmem_schedule(sk, skb, size)) {
4198 4199 4200 4201

		if (tcp_prune_queue(sk) < 0)
			return -1;

4202
		if (!sk_rmem_schedule(sk, skb, size)) {
4203 4204 4205
			if (!tcp_prune_ofo_queue(sk))
				return -1;

4206
			if (!sk_rmem_schedule(sk, skb, size))
4207 4208 4209 4210 4211 4212
				return -1;
		}
	}
	return 0;
}

E
Eric Dumazet 已提交
4213 4214 4215 4216 4217 4218
static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb1;
	u32 seq, end_seq;

4219
	tcp_ecn_check_ce(tp, skb);
E
Eric Dumazet 已提交
4220

4221
	if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) {
E
Eric Dumazet 已提交
4222
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFODROP);
E
Eric Dumazet 已提交
4223 4224 4225 4226 4227 4228 4229 4230
		__kfree_skb(skb);
		return;
	}

	/* Disable header prediction. */
	tp->pred_flags = 0;
	inet_csk_schedule_ack(sk);

E
Eric Dumazet 已提交
4231
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOQUEUE);
E
Eric Dumazet 已提交
4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251
	SOCK_DEBUG(sk, "out of order segment: rcv_next %X seq %X - %X\n",
		   tp->rcv_nxt, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);

	skb1 = skb_peek_tail(&tp->out_of_order_queue);
	if (!skb1) {
		/* Initial out of order segment, build 1 SACK. */
		if (tcp_is_sack(tp)) {
			tp->rx_opt.num_sacks = 1;
			tp->selective_acks[0].start_seq = TCP_SKB_CB(skb)->seq;
			tp->selective_acks[0].end_seq =
						TCP_SKB_CB(skb)->end_seq;
		}
		__skb_queue_head(&tp->out_of_order_queue, skb);
		goto end;
	}

	seq = TCP_SKB_CB(skb)->seq;
	end_seq = TCP_SKB_CB(skb)->end_seq;

	if (seq == TCP_SKB_CB(skb1)->end_seq) {
4252 4253 4254
		bool fragstolen;

		if (!tcp_try_coalesce(sk, skb1, skb, &fragstolen)) {
E
Eric Dumazet 已提交
4255 4256
			__skb_queue_after(&tp->out_of_order_queue, skb1, skb);
		} else {
4257
			tcp_grow_window(sk, skb);
4258
			kfree_skb_partial(skb, fragstolen);
4259 4260
			skb = NULL;
		}
E
Eric Dumazet 已提交
4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285

		if (!tp->rx_opt.num_sacks ||
		    tp->selective_acks[0].end_seq != seq)
			goto add_sack;

		/* Common case: data arrive in order after hole. */
		tp->selective_acks[0].end_seq = end_seq;
		goto end;
	}

	/* Find place to insert this segment. */
	while (1) {
		if (!after(TCP_SKB_CB(skb1)->seq, seq))
			break;
		if (skb_queue_is_first(&tp->out_of_order_queue, skb1)) {
			skb1 = NULL;
			break;
		}
		skb1 = skb_queue_prev(&tp->out_of_order_queue, skb1);
	}

	/* Do skb overlap to previous one? */
	if (skb1 && before(seq, TCP_SKB_CB(skb1)->end_seq)) {
		if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
			/* All the bits are present. Drop. */
E
Eric Dumazet 已提交
4286
			NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
E
Eric Dumazet 已提交
4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324
			__kfree_skb(skb);
			skb = NULL;
			tcp_dsack_set(sk, seq, end_seq);
			goto add_sack;
		}
		if (after(seq, TCP_SKB_CB(skb1)->seq)) {
			/* Partial overlap. */
			tcp_dsack_set(sk, seq,
				      TCP_SKB_CB(skb1)->end_seq);
		} else {
			if (skb_queue_is_first(&tp->out_of_order_queue,
					       skb1))
				skb1 = NULL;
			else
				skb1 = skb_queue_prev(
					&tp->out_of_order_queue,
					skb1);
		}
	}
	if (!skb1)
		__skb_queue_head(&tp->out_of_order_queue, skb);
	else
		__skb_queue_after(&tp->out_of_order_queue, skb1, skb);

	/* And clean segments covered by new one as whole. */
	while (!skb_queue_is_last(&tp->out_of_order_queue, skb)) {
		skb1 = skb_queue_next(&tp->out_of_order_queue, skb);

		if (!after(end_seq, TCP_SKB_CB(skb1)->seq))
			break;
		if (before(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
			tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
					 end_seq);
			break;
		}
		__skb_unlink(skb1, &tp->out_of_order_queue);
		tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
				 TCP_SKB_CB(skb1)->end_seq);
E
Eric Dumazet 已提交
4325
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
E
Eric Dumazet 已提交
4326 4327 4328 4329 4330 4331 4332
		__kfree_skb(skb1);
	}

add_sack:
	if (tcp_is_sack(tp))
		tcp_sack_new_ofo_skb(sk, seq, end_seq);
end:
4333 4334
	if (skb) {
		tcp_grow_window(sk, skb);
E
Eric Dumazet 已提交
4335
		skb_set_owner_r(skb, sk);
4336
	}
E
Eric Dumazet 已提交
4337 4338
}

4339
static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int hdrlen,
4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354
		  bool *fragstolen)
{
	int eaten;
	struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue);

	__skb_pull(skb, hdrlen);
	eaten = (tail &&
		 tcp_try_coalesce(sk, tail, skb, fragstolen)) ? 1 : 0;
	tcp_sk(sk)->rcv_nxt = TCP_SKB_CB(skb)->end_seq;
	if (!eaten) {
		__skb_queue_tail(&sk->sk_receive_queue, skb);
		skb_set_owner_r(skb, sk);
	}
	return eaten;
}
E
Eric Dumazet 已提交
4355

4356 4357
int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)
{
4358
	struct sk_buff *skb;
4359 4360
	bool fragstolen;

4361 4362 4363
	if (size == 0)
		return 0;

4364
	skb = alloc_skb(size, sk->sk_allocation);
4365 4366 4367
	if (!skb)
		goto err;

4368
	if (tcp_try_rmem_schedule(sk, skb, skb->truesize))
4369 4370
		goto err_free;

A
Al Viro 已提交
4371
	if (memcpy_from_msg(skb_put(skb, size), msg, size))
4372 4373 4374 4375 4376 4377
		goto err_free;

	TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt;
	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size;
	TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1;

4378
	if (tcp_queue_rcv(sk, skb, 0, &fragstolen)) {
4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389
		WARN_ON_ONCE(fragstolen); /* should not happen */
		__kfree_skb(skb);
	}
	return size;

err_free:
	kfree_skb(skb);
err:
	return -ENOMEM;
}

L
Linus Torvalds 已提交
4390 4391 4392 4393
static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);
	int eaten = -1;
4394
	bool fragstolen = false;
L
Linus Torvalds 已提交
4395 4396 4397 4398

	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq)
		goto drop;

E
Eric Dumazet 已提交
4399
	skb_dst_drop(skb);
4400
	__skb_pull(skb, tcp_hdr(skb)->doff * 4);
L
Linus Torvalds 已提交
4401

4402
	tcp_ecn_accept_cwr(tp, skb);
L
Linus Torvalds 已提交
4403

4404
	tp->rx_opt.dsack = 0;
L
Linus Torvalds 已提交
4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418

	/*  Queue data for delivery to the user.
	 *  Packets in sequence go to the receive queue.
	 *  Out of sequence packets to the out_of_order_queue.
	 */
	if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) {
		if (tcp_receive_window(tp) == 0)
			goto out_of_window;

		/* Ok. In sequence. In window. */
		if (tp->ucopy.task == current &&
		    tp->copied_seq == tp->rcv_nxt && tp->ucopy.len &&
		    sock_owned_by_user(sk) && !tp->urg_data) {
			int chunk = min_t(unsigned int, skb->len,
4419
					  tp->ucopy.len);
L
Linus Torvalds 已提交
4420 4421 4422 4423

			__set_current_state(TASK_RUNNING);

			local_bh_enable();
4424
			if (!skb_copy_datagram_msg(skb, 0, tp->ucopy.msg, chunk)) {
L
Linus Torvalds 已提交
4425 4426
				tp->ucopy.len -= chunk;
				tp->copied_seq += chunk;
4427
				eaten = (chunk == skb->len);
L
Linus Torvalds 已提交
4428 4429 4430 4431 4432 4433 4434 4435
				tcp_rcv_space_adjust(sk);
			}
			local_bh_disable();
		}

		if (eaten <= 0) {
queue_and_out:
			if (eaten < 0 &&
4436
			    tcp_try_rmem_schedule(sk, skb, skb->truesize))
4437 4438
				goto drop;

4439
			eaten = tcp_queue_rcv(sk, skb, 0, &fragstolen);
L
Linus Torvalds 已提交
4440 4441
		}
		tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq;
S
Stephen Hemminger 已提交
4442
		if (skb->len)
4443
			tcp_event_data_recv(sk, skb);
4444
		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
4445
			tcp_fin(sk);
L
Linus Torvalds 已提交
4446

4447
		if (!skb_queue_empty(&tp->out_of_order_queue)) {
L
Linus Torvalds 已提交
4448 4449 4450 4451 4452
			tcp_ofo_queue(sk);

			/* RFC2581. 4.2. SHOULD send immediate ACK, when
			 * gap in queue is filled.
			 */
4453
			if (skb_queue_empty(&tp->out_of_order_queue))
4454
				inet_csk(sk)->icsk_ack.pingpong = 0;
L
Linus Torvalds 已提交
4455 4456 4457 4458 4459
		}

		if (tp->rx_opt.num_sacks)
			tcp_sack_remove(tp);

4460
		tcp_fast_path_check(sk);
L
Linus Torvalds 已提交
4461

4462 4463
		if (eaten > 0)
			kfree_skb_partial(skb, fragstolen);
4464
		if (!sock_flag(sk, SOCK_DEAD))
4465
			sk->sk_data_ready(sk);
L
Linus Torvalds 已提交
4466 4467 4468 4469 4470
		return;
	}

	if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
		/* A retransmit, 2nd most common case.  Force an immediate ack. */
4471
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
4472
		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
L
Linus Torvalds 已提交
4473 4474

out_of_window:
4475 4476
		tcp_enter_quickack_mode(sk);
		inet_csk_schedule_ack(sk);
L
Linus Torvalds 已提交
4477 4478 4479 4480 4481 4482 4483 4484 4485
drop:
		__kfree_skb(skb);
		return;
	}

	/* Out of window. F.e. zero window probe. */
	if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt + tcp_receive_window(tp)))
		goto out_of_window;

4486
	tcp_enter_quickack_mode(sk);
L
Linus Torvalds 已提交
4487 4488 4489 4490 4491 4492 4493

	if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
		/* Partial packet, seq < rcv_next < end_seq */
		SOCK_DEBUG(sk, "partial packet: rcv_next %X seq %X - %X\n",
			   tp->rcv_nxt, TCP_SKB_CB(skb)->seq,
			   TCP_SKB_CB(skb)->end_seq);

4494
		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, tp->rcv_nxt);
4495

L
Linus Torvalds 已提交
4496 4497 4498 4499 4500 4501 4502 4503
		/* If window is closed, drop tail of packet. But after
		 * remembering D-SACK for its head made in previous line.
		 */
		if (!tcp_receive_window(tp))
			goto out_of_window;
		goto queue_and_out;
	}

E
Eric Dumazet 已提交
4504
	tcp_data_queue_ofo(sk, skb);
L
Linus Torvalds 已提交
4505 4506
}

4507 4508 4509
static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb,
					struct sk_buff_head *list)
{
4510 4511 4512 4513
	struct sk_buff *next = NULL;

	if (!skb_queue_is_last(list, skb))
		next = skb_queue_next(list, skb);
4514 4515 4516 4517 4518 4519 4520 4521

	__skb_unlink(skb, list);
	__kfree_skb(skb);
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED);

	return next;
}

L
Linus Torvalds 已提交
4522 4523
/* Collapse contiguous sequence of skbs head..tail with
 * sequence numbers start..end.
4524 4525 4526
 *
 * If tail is NULL, this means until the end of the list.
 *
L
Linus Torvalds 已提交
4527 4528 4529 4530
 * Segments with FIN/SYN are not collapsed (only because this
 * simplifies code)
 */
static void
D
David S. Miller 已提交
4531 4532 4533
tcp_collapse(struct sock *sk, struct sk_buff_head *list,
	     struct sk_buff *head, struct sk_buff *tail,
	     u32 start, u32 end)
L
Linus Torvalds 已提交
4534
{
4535 4536
	struct sk_buff *skb, *n;
	bool end_of_skbs;
L
Linus Torvalds 已提交
4537

S
Stephen Hemminger 已提交
4538
	/* First, check that queue is collapsible and find
L
Linus Torvalds 已提交
4539
	 * the point where collapsing can be useful. */
4540 4541 4542 4543 4544 4545
	skb = head;
restart:
	end_of_skbs = true;
	skb_queue_walk_from_safe(list, skb, n) {
		if (skb == tail)
			break;
L
Linus Torvalds 已提交
4546 4547
		/* No new bits? It is possible on ofo queue. */
		if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
4548
			skb = tcp_collapse_one(sk, skb, list);
4549 4550 4551
			if (!skb)
				break;
			goto restart;
L
Linus Torvalds 已提交
4552 4553 4554 4555 4556 4557 4558
		}

		/* The first skb to collapse is:
		 * - not SYN/FIN and
		 * - bloated or contains data before "start" or
		 *   overlaps to the next one.
		 */
4559
		if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&
L
Linus Torvalds 已提交
4560
		    (tcp_win_from_space(skb->truesize) > skb->len ||
4561 4562
		     before(TCP_SKB_CB(skb)->seq, start))) {
			end_of_skbs = false;
L
Linus Torvalds 已提交
4563
			break;
4564 4565 4566 4567 4568 4569 4570 4571 4572 4573
		}

		if (!skb_queue_is_last(list, skb)) {
			struct sk_buff *next = skb_queue_next(list, skb);
			if (next != tail &&
			    TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(next)->seq) {
				end_of_skbs = false;
				break;
			}
		}
L
Linus Torvalds 已提交
4574 4575 4576 4577

		/* Decided to skip this, advance start seq. */
		start = TCP_SKB_CB(skb)->end_seq;
	}
4578 4579
	if (end_of_skbs ||
	    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
L
Linus Torvalds 已提交
4580 4581 4582
		return;

	while (before(start, end)) {
4583
		int copy = min_t(int, SKB_MAX_ORDER(0, 0), end - start);
L
Linus Torvalds 已提交
4584 4585
		struct sk_buff *nskb;

4586
		nskb = alloc_skb(copy, GFP_ATOMIC);
L
Linus Torvalds 已提交
4587 4588
		if (!nskb)
			return;
4589

L
Linus Torvalds 已提交
4590 4591
		memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
		TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
4592
		__skb_queue_before(list, skb, nskb);
4593
		skb_set_owner_r(nskb, sk);
L
Linus Torvalds 已提交
4594 4595 4596 4597 4598 4599

		/* Copy data, releasing collapsed skbs. */
		while (copy > 0) {
			int offset = start - TCP_SKB_CB(skb)->seq;
			int size = TCP_SKB_CB(skb)->end_seq - start;

4600
			BUG_ON(offset < 0);
L
Linus Torvalds 已提交
4601 4602 4603 4604 4605 4606 4607 4608 4609
			if (size > 0) {
				size = min(copy, size);
				if (skb_copy_bits(skb, offset, skb_put(nskb, size), size))
					BUG();
				TCP_SKB_CB(nskb)->end_seq += size;
				copy -= size;
				start += size;
			}
			if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
4610
				skb = tcp_collapse_one(sk, skb, list);
4611 4612
				if (!skb ||
				    skb == tail ||
4613
				    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
L
Linus Torvalds 已提交
4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637
					return;
			}
		}
	}
}

/* Collapse ofo queue. Algorithm: select contiguous sequence of skbs
 * and tcp_collapse() them until all the queue is collapsed.
 */
static void tcp_collapse_ofo_queue(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb = skb_peek(&tp->out_of_order_queue);
	struct sk_buff *head;
	u32 start, end;

	if (skb == NULL)
		return;

	start = TCP_SKB_CB(skb)->seq;
	end = TCP_SKB_CB(skb)->end_seq;
	head = skb;

	for (;;) {
4638 4639 4640 4641 4642
		struct sk_buff *next = NULL;

		if (!skb_queue_is_last(&tp->out_of_order_queue, skb))
			next = skb_queue_next(&tp->out_of_order_queue, skb);
		skb = next;
L
Linus Torvalds 已提交
4643 4644 4645

		/* Segment is terminated when we see gap or when
		 * we are at the end of all the queue. */
4646
		if (!skb ||
L
Linus Torvalds 已提交
4647 4648
		    after(TCP_SKB_CB(skb)->seq, end) ||
		    before(TCP_SKB_CB(skb)->end_seq, start)) {
D
David S. Miller 已提交
4649 4650
			tcp_collapse(sk, &tp->out_of_order_queue,
				     head, skb, start, end);
L
Linus Torvalds 已提交
4651
			head = skb;
4652
			if (!skb)
L
Linus Torvalds 已提交
4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665
				break;
			/* Start new segment */
			start = TCP_SKB_CB(skb)->seq;
			end = TCP_SKB_CB(skb)->end_seq;
		} else {
			if (before(TCP_SKB_CB(skb)->seq, start))
				start = TCP_SKB_CB(skb)->seq;
			if (after(TCP_SKB_CB(skb)->end_seq, end))
				end = TCP_SKB_CB(skb)->end_seq;
		}
	}
}

4666 4667
/*
 * Purge the out-of-order queue.
4668
 * Return true if queue was pruned.
4669
 */
E
Eric Dumazet 已提交
4670
static bool tcp_prune_ofo_queue(struct sock *sk)
4671 4672
{
	struct tcp_sock *tp = tcp_sk(sk);
E
Eric Dumazet 已提交
4673
	bool res = false;
4674 4675

	if (!skb_queue_empty(&tp->out_of_order_queue)) {
4676
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_OFOPRUNED);
4677 4678 4679 4680 4681 4682 4683 4684 4685 4686
		__skb_queue_purge(&tp->out_of_order_queue);

		/* Reset SACK state.  A conforming SACK implementation will
		 * do the same at a timeout based retransmit.  When a connection
		 * is in a sad state like this, we care only about integrity
		 * of the connection not performance.
		 */
		if (tp->rx_opt.sack_ok)
			tcp_sack_reset(&tp->rx_opt);
		sk_mem_reclaim(sk);
E
Eric Dumazet 已提交
4687
		res = true;
4688
	}
4689
	return res;
4690 4691
}

L
Linus Torvalds 已提交
4692 4693 4694 4695 4696 4697 4698 4699 4700
/* Reduce allocated memory if we can, trying to get
 * the socket within its memory limits again.
 *
 * Return less than zero if we should start dropping frames
 * until the socket owning process reads some of the data
 * to stabilize the situation.
 */
static int tcp_prune_queue(struct sock *sk)
{
4701
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
4702 4703 4704

	SOCK_DEBUG(sk, "prune_queue: c=%x\n", tp->copied_seq);

4705
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PRUNECALLED);
L
Linus Torvalds 已提交
4706 4707

	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
4708
		tcp_clamp_window(sk);
4709
	else if (sk_under_memory_pressure(sk))
L
Linus Torvalds 已提交
4710 4711 4712
		tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss);

	tcp_collapse_ofo_queue(sk);
4713 4714 4715 4716 4717
	if (!skb_queue_empty(&sk->sk_receive_queue))
		tcp_collapse(sk, &sk->sk_receive_queue,
			     skb_peek(&sk->sk_receive_queue),
			     NULL,
			     tp->copied_seq, tp->rcv_nxt);
4718
	sk_mem_reclaim(sk);
L
Linus Torvalds 已提交
4719 4720 4721 4722 4723 4724 4725

	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
		return 0;

	/* Collapsing did not help, destructive actions follow.
	 * This must not ever occur. */

4726
	tcp_prune_ofo_queue(sk);
L
Linus Torvalds 已提交
4727 4728 4729 4730 4731 4732 4733 4734

	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
		return 0;

	/* If we are really being abused, tell the caller to silently
	 * drop receive data on the floor.  It will get retransmitted
	 * and hopefully then we'll have sufficient space.
	 */
4735
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_RCVPRUNED);
L
Linus Torvalds 已提交
4736 4737 4738 4739 4740 4741

	/* Massive buffer overcommit. */
	tp->pred_flags = 0;
	return -1;
}

E
Eric Dumazet 已提交
4742
static bool tcp_should_expand_sndbuf(const struct sock *sk)
4743
{
4744
	const struct tcp_sock *tp = tcp_sk(sk);
4745

4746 4747 4748 4749
	/* If the user specified a specific send buffer setting, do
	 * not modify it.
	 */
	if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
E
Eric Dumazet 已提交
4750
		return false;
4751 4752

	/* If we are under global TCP memory pressure, do not expand.  */
4753
	if (sk_under_memory_pressure(sk))
E
Eric Dumazet 已提交
4754
		return false;
4755 4756

	/* If we are under soft global TCP memory pressure, do not expand.  */
4757
	if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, 0))
E
Eric Dumazet 已提交
4758
		return false;
4759 4760 4761

	/* If we filled the congestion window, do not expand.  */
	if (tp->packets_out >= tp->snd_cwnd)
E
Eric Dumazet 已提交
4762
		return false;
4763

E
Eric Dumazet 已提交
4764
	return true;
4765
}
L
Linus Torvalds 已提交
4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776

/* When incoming ACK allowed to free some skb from write_queue,
 * we remember this event in flag SOCK_QUEUE_SHRUNK and wake up socket
 * on the exit from tcp input handler.
 *
 * PROBLEM: sndbuf expansion does not work well with largesend.
 */
static void tcp_new_space(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);

4777
	if (tcp_should_expand_sndbuf(sk)) {
E
Eric Dumazet 已提交
4778
		tcp_sndbuf_expand(sk);
L
Linus Torvalds 已提交
4779 4780 4781 4782 4783 4784
		tp->snd_cwnd_stamp = tcp_time_stamp;
	}

	sk->sk_write_space(sk);
}

S
Stephen Hemminger 已提交
4785
static void tcp_check_space(struct sock *sk)
L
Linus Torvalds 已提交
4786 4787 4788 4789 4790 4791 4792 4793 4794
{
	if (sock_flag(sk, SOCK_QUEUE_SHRUNK)) {
		sock_reset_flag(sk, SOCK_QUEUE_SHRUNK);
		if (sk->sk_socket &&
		    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
			tcp_new_space(sk);
	}
}

4795
static inline void tcp_data_snd_check(struct sock *sk)
L
Linus Torvalds 已提交
4796
{
4797
	tcp_push_pending_frames(sk);
L
Linus Torvalds 已提交
4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808
	tcp_check_space(sk);
}

/*
 * Check if sending an ack is needed.
 */
static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
{
	struct tcp_sock *tp = tcp_sk(sk);

	    /* More than one full frame received... */
4809
	if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&
L
Linus Torvalds 已提交
4810 4811 4812
	     /* ... and right edge of window advances far enough.
	      * (tcp_recvmsg() will send ACK otherwise). Or...
	      */
4813
	     __tcp_select_window(sk) >= tp->rcv_wnd) ||
L
Linus Torvalds 已提交
4814
	    /* We ACK each frame or... */
4815
	    tcp_in_quickack_mode(sk) ||
L
Linus Torvalds 已提交
4816
	    /* We have out of order data. */
4817
	    (ofo_possible && skb_peek(&tp->out_of_order_queue))) {
L
Linus Torvalds 已提交
4818 4819 4820 4821 4822 4823 4824 4825
		/* Then ack it now */
		tcp_send_ack(sk);
	} else {
		/* Else, send delayed ack. */
		tcp_send_delayed_ack(sk);
	}
}

S
Stephen Hemminger 已提交
4826
static inline void tcp_ack_snd_check(struct sock *sk)
L
Linus Torvalds 已提交
4827
{
4828
	if (!inet_csk_ack_scheduled(sk)) {
L
Linus Torvalds 已提交
4829 4830 4831 4832 4833 4834 4835 4836
		/* We sent a data segment already. */
		return;
	}
	__tcp_ack_snd_check(sk, 1);
}

/*
 *	This routine is only called when we have urgent data
S
Stephen Hemminger 已提交
4837
 *	signaled. Its the 'slow' part of tcp_urg. It could be
L
Linus Torvalds 已提交
4838 4839 4840 4841 4842 4843
 *	moved inline now as tcp_urg is only called from one
 *	place. We handle URGent data wrong. We have to - as
 *	BSD still doesn't use the correction from RFC961.
 *	For 1003.1g we should support a new option TCP_STDURG to permit
 *	either form (or just set the sysctl tcp_stdurg).
 */
4844

4845
static void tcp_check_urg(struct sock *sk, const struct tcphdr *th)
L
Linus Torvalds 已提交
4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881
{
	struct tcp_sock *tp = tcp_sk(sk);
	u32 ptr = ntohs(th->urg_ptr);

	if (ptr && !sysctl_tcp_stdurg)
		ptr--;
	ptr += ntohl(th->seq);

	/* Ignore urgent data that we've already seen and read. */
	if (after(tp->copied_seq, ptr))
		return;

	/* Do not replay urg ptr.
	 *
	 * NOTE: interesting situation not covered by specs.
	 * Misbehaving sender may send urg ptr, pointing to segment,
	 * which we already have in ofo queue. We are not able to fetch
	 * such data and will stay in TCP_URG_NOTYET until will be eaten
	 * by recvmsg(). Seems, we are not obliged to handle such wicked
	 * situations. But it is worth to think about possibility of some
	 * DoSes using some hypothetical application level deadlock.
	 */
	if (before(ptr, tp->rcv_nxt))
		return;

	/* Do we already have a newer (or duplicate) urgent pointer? */
	if (tp->urg_data && !after(ptr, tp->urg_seq))
		return;

	/* Tell the world about our new urgent pointer. */
	sk_send_sigurg(sk);

	/* We may be adding urgent data when the last byte read was
	 * urgent. To do this requires some care. We cannot just ignore
	 * tp->copied_seq since we would read the last urgent byte again
	 * as data, nor can we alter copied_seq until this data arrives
S
Stephen Hemminger 已提交
4882
	 * or we break the semantics of SIOCATMARK (and thus sockatmark())
L
Linus Torvalds 已提交
4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893
	 *
	 * NOTE. Double Dutch. Rendering to plain English: author of comment
	 * above did something sort of 	send("A", MSG_OOB); send("B", MSG_OOB);
	 * and expect that both A and B disappear from stream. This is _wrong_.
	 * Though this happens in BSD with high probability, this is occasional.
	 * Any application relying on this is buggy. Note also, that fix "works"
	 * only in this artificial test. Insert some normal data between A and B and we will
	 * decline of BSD again. Verdict: it is better to remove to trap
	 * buggy users.
	 */
	if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
4894
	    !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
L
Linus Torvalds 已提交
4895 4896 4897
		struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
		tp->copied_seq++;
		if (skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) {
D
David S. Miller 已提交
4898
			__skb_unlink(skb, &sk->sk_receive_queue);
L
Linus Torvalds 已提交
4899 4900 4901 4902
			__kfree_skb(skb);
		}
	}

4903 4904
	tp->urg_data = TCP_URG_NOTYET;
	tp->urg_seq = ptr;
L
Linus Torvalds 已提交
4905 4906 4907 4908 4909 4910

	/* Disable header prediction. */
	tp->pred_flags = 0;
}

/* This is the 'fast' part of urgent handling. */
4911
static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th)
L
Linus Torvalds 已提交
4912 4913 4914 4915 4916
{
	struct tcp_sock *tp = tcp_sk(sk);

	/* Check if we get a new urgent pointer - normally not. */
	if (th->urg)
4917
		tcp_check_urg(sk, th);
L
Linus Torvalds 已提交
4918 4919 4920 4921 4922 4923

	/* Do we wait for any urgent data? - normally not... */
	if (tp->urg_data == TCP_URG_NOTYET) {
		u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) -
			  th->syn;

4924
		/* Is the urgent pointer pointing into this packet? */
L
Linus Torvalds 已提交
4925 4926 4927 4928 4929 4930
		if (ptr < skb->len) {
			u8 tmp;
			if (skb_copy_bits(skb, ptr, &tmp, 1))
				BUG();
			tp->urg_data = TCP_URG_VALID | tmp;
			if (!sock_flag(sk, SOCK_DEAD))
4931
				sk->sk_data_ready(sk);
L
Linus Torvalds 已提交
4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942
		}
	}
}

static int tcp_copy_to_iovec(struct sock *sk, struct sk_buff *skb, int hlen)
{
	struct tcp_sock *tp = tcp_sk(sk);
	int chunk = skb->len - hlen;
	int err;

	local_bh_enable();
4943
	if (skb_csum_unnecessary(skb))
4944
		err = skb_copy_datagram_msg(skb, hlen, tp->ucopy.msg, chunk);
L
Linus Torvalds 已提交
4945
	else
4946
		err = skb_copy_and_csum_datagram_msg(skb, hlen, tp->ucopy.msg);
L
Linus Torvalds 已提交
4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957

	if (!err) {
		tp->ucopy.len -= chunk;
		tp->copied_seq += chunk;
		tcp_rcv_space_adjust(sk);
	}

	local_bh_disable();
	return err;
}

4958 4959
static __sum16 __tcp_checksum_complete_user(struct sock *sk,
					    struct sk_buff *skb)
L
Linus Torvalds 已提交
4960
{
4961
	__sum16 result;
L
Linus Torvalds 已提交
4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972

	if (sock_owned_by_user(sk)) {
		local_bh_enable();
		result = __tcp_checksum_complete(skb);
		local_bh_disable();
	} else {
		result = __tcp_checksum_complete(skb);
	}
	return result;
}

4973
static inline bool tcp_checksum_complete_user(struct sock *sk,
4974
					     struct sk_buff *skb)
L
Linus Torvalds 已提交
4975
{
4976
	return !skb_csum_unnecessary(skb) &&
4977
	       __tcp_checksum_complete_user(sk, skb);
L
Linus Torvalds 已提交
4978 4979
}

4980 4981 4982
/* Does PAWS and seqno based validation of an incoming segment, flags will
 * play significant role here.
 */
E
Eric Dumazet 已提交
4983 4984
static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
				  const struct tcphdr *th, int syn_inerr)
4985 4986 4987 4988
{
	struct tcp_sock *tp = tcp_sk(sk);

	/* RFC1323: H1. Apply PAWS check first. */
C
Christoph Paasch 已提交
4989
	if (tcp_fast_parse_options(skb, th, tp) && tp->rx_opt.saw_tstamp &&
4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006
	    tcp_paws_discard(sk, skb)) {
		if (!th->rst) {
			NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
			tcp_send_dupack(sk, skb);
			goto discard;
		}
		/* Reset is accepted even if it did not pass PAWS. */
	}

	/* Step 1: check sequence number */
	if (!tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)) {
		/* RFC793, page 37: "In all states except SYN-SENT, all reset
		 * (RST) segments are validated by checking their SEQ-fields."
		 * And page 69: "If an incoming segment is not acceptable,
		 * an acknowledgment should be sent in reply (unless the RST
		 * bit is set, if so drop the segment and return)".
		 */
5007 5008 5009
		if (!th->rst) {
			if (th->syn)
				goto syn_challenge;
5010
			tcp_send_dupack(sk, skb);
5011
		}
5012 5013 5014 5015 5016
		goto discard;
	}

	/* Step 2: check RST bit */
	if (th->rst) {
E
Eric Dumazet 已提交
5017 5018 5019 5020 5021 5022 5023 5024 5025 5026
		/* RFC 5961 3.2 :
		 * If sequence number exactly matches RCV.NXT, then
		 *     RESET the connection
		 * else
		 *     Send a challenge ACK
		 */
		if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt)
			tcp_reset(sk);
		else
			tcp_send_challenge_ack(sk);
5027 5028 5029 5030 5031
		goto discard;
	}

	/* step 3: check security and precedence [ignored] */

E
Eric Dumazet 已提交
5032
	/* step 4: Check for a SYN
5033
	 * RFC 5961 4.2 : Send a challenge ack
E
Eric Dumazet 已提交
5034 5035
	 */
	if (th->syn) {
5036
syn_challenge:
5037 5038
		if (syn_inerr)
			TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS);
E
Eric Dumazet 已提交
5039 5040 5041
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE);
		tcp_send_challenge_ack(sk);
		goto discard;
5042 5043
	}

E
Eric Dumazet 已提交
5044
	return true;
5045 5046 5047

discard:
	__kfree_skb(skb);
E
Eric Dumazet 已提交
5048
	return false;
5049 5050
}

L
Linus Torvalds 已提交
5051
/*
5052
 *	TCP receive function for the ESTABLISHED state.
L
Linus Torvalds 已提交
5053
 *
5054
 *	It is split into a fast path and a slow path. The fast path is
L
Linus Torvalds 已提交
5055 5056
 * 	disabled when:
 *	- A zero window was announced from us - zero window probing
5057
 *        is only handled properly in the slow path.
L
Linus Torvalds 已提交
5058 5059 5060 5061
 *	- Out of order segments arrived.
 *	- Urgent data is expected.
 *	- There is no buffer space left
 *	- Unexpected TCP flags/window values/header lengths are received
5062
 *	  (detected by checking the TCP header against pred_flags)
L
Linus Torvalds 已提交
5063 5064 5065 5066 5067
 *	- Data is sent in both directions. Fast path only supports pure senders
 *	  or pure receivers (this means either the sequence number or the ack
 *	  value must stay constant)
 *	- Unexpected TCP option.
 *
5068
 *	When these conditions are not satisfied it drops into a standard
L
Linus Torvalds 已提交
5069 5070
 *	receive procedure patterned after RFC793 to handle all cases.
 *	The first three cases are guaranteed by proper pred_flags setting,
5071
 *	the rest is checked inline. Fast processing is turned on in
L
Linus Torvalds 已提交
5072 5073
 *	tcp_data_queue when everything is OK.
 */
5074 5075
void tcp_rcv_established(struct sock *sk, struct sk_buff *skb,
			 const struct tcphdr *th, unsigned int len)
L
Linus Torvalds 已提交
5076 5077 5078
{
	struct tcp_sock *tp = tcp_sk(sk);

E
Eric Dumazet 已提交
5079 5080
	if (unlikely(sk->sk_rx_dst == NULL))
		inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
L
Linus Torvalds 已提交
5081 5082
	/*
	 *	Header prediction.
5083
	 *	The code loosely follows the one in the famous
L
Linus Torvalds 已提交
5084
	 *	"30 instruction TCP receive" Van Jacobson mail.
5085 5086
	 *
	 *	Van's trick is to deposit buffers into socket queue
L
Linus Torvalds 已提交
5087 5088 5089 5090
	 *	on a device interrupt, to call tcp_recv function
	 *	on the receive process context and checksum and copy
	 *	the buffer to user space. smart...
	 *
5091
	 *	Our current scheme is not silly either but we take the
L
Linus Torvalds 已提交
5092 5093 5094 5095 5096 5097 5098
	 *	extra cost of the net_bh soft interrupt processing...
	 *	We do checksum and copy also but from device to kernel.
	 */

	tp->rx_opt.saw_tstamp = 0;

	/*	pred_flags is 0xS?10 << 16 + snd_wnd
S
Stephen Hemminger 已提交
5099
	 *	if header_prediction is to be made
L
Linus Torvalds 已提交
5100 5101
	 *	'S' will always be tp->tcp_header_len >> 2
	 *	'?' will be 0 for the fast path, otherwise pred_flags is 0 to
5102
	 *  turn it off	(when there are holes in the receive
L
Linus Torvalds 已提交
5103 5104 5105 5106 5107
	 *	 space for instance)
	 *	PSH flag is ignored.
	 */

	if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags &&
5108 5109
	    TCP_SKB_CB(skb)->seq == tp->rcv_nxt &&
	    !after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
L
Linus Torvalds 已提交
5110 5111 5112 5113 5114 5115 5116 5117 5118 5119
		int tcp_header_len = tp->tcp_header_len;

		/* Timestamp header prediction: tcp_header_len
		 * is automatically equal to th->doff*4 due to pred_flags
		 * match.
		 */

		/* Check timestamp */
		if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) {
			/* No? Slow path! */
5120
			if (!tcp_parse_aligned_timestamp(tp, th))
L
Linus Torvalds 已提交
5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149
				goto slow_path;

			/* If PAWS failed, check it more carefully in slow path */
			if ((s32)(tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent) < 0)
				goto slow_path;

			/* DO NOT update ts_recent here, if checksum fails
			 * and timestamp was corrupted part, it will result
			 * in a hung connection since we will drop all
			 * future packets due to the PAWS test.
			 */
		}

		if (len <= tcp_header_len) {
			/* Bulk data transfer: sender */
			if (len == tcp_header_len) {
				/* Predicted packet is in window by definition.
				 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
				 * Hence, check seq<=rcv_wup reduces to:
				 */
				if (tcp_header_len ==
				    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
				    tp->rcv_nxt == tp->rcv_wup)
					tcp_store_ts_recent(tp);

				/* We know that such packets are checksummed
				 * on entry.
				 */
				tcp_ack(sk, skb, 0);
5150
				__kfree_skb(skb);
5151
				tcp_data_snd_check(sk);
5152
				return;
L
Linus Torvalds 已提交
5153
			} else { /* Header too small */
5154
				TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS);
L
Linus Torvalds 已提交
5155 5156 5157 5158
				goto discard;
			}
		} else {
			int eaten = 0;
5159
			bool fragstolen = false;
L
Linus Torvalds 已提交
5160

D
Dan Williams 已提交
5161 5162 5163 5164 5165
			if (tp->ucopy.task == current &&
			    tp->copied_seq == tp->rcv_nxt &&
			    len - tcp_header_len <= tp->ucopy.len &&
			    sock_owned_by_user(sk)) {
				__set_current_state(TASK_RUNNING);
L
Linus Torvalds 已提交
5166

D
Dan Williams 已提交
5167
				if (!tcp_copy_to_iovec(sk, skb, tcp_header_len)) {
L
Linus Torvalds 已提交
5168 5169 5170 5171 5172 5173 5174 5175 5176 5177
					/* Predicted packet is in window by definition.
					 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
					 * Hence, check seq<=rcv_wup reduces to:
					 */
					if (tcp_header_len ==
					    (sizeof(struct tcphdr) +
					     TCPOLEN_TSTAMP_ALIGNED) &&
					    tp->rcv_nxt == tp->rcv_wup)
						tcp_store_ts_recent(tp);

5178
					tcp_rcv_rtt_measure_ts(sk, skb);
L
Linus Torvalds 已提交
5179 5180 5181

					__skb_pull(skb, tcp_header_len);
					tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq;
5182
					NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPHITSTOUSER);
D
Dan Williams 已提交
5183
					eaten = 1;
L
Linus Torvalds 已提交
5184 5185 5186 5187 5188 5189
				}
			}
			if (!eaten) {
				if (tcp_checksum_complete_user(sk, skb))
					goto csum_error;

5190 5191 5192
				if ((int)skb->truesize > sk->sk_forward_alloc)
					goto step5;

L
Linus Torvalds 已提交
5193 5194 5195 5196 5197 5198 5199 5200 5201
				/* Predicted packet is in window by definition.
				 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
				 * Hence, check seq<=rcv_wup reduces to:
				 */
				if (tcp_header_len ==
				    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
				    tp->rcv_nxt == tp->rcv_wup)
					tcp_store_ts_recent(tp);

5202
				tcp_rcv_rtt_measure_ts(sk, skb);
L
Linus Torvalds 已提交
5203

5204
				NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPHPHITS);
L
Linus Torvalds 已提交
5205 5206

				/* Bulk data transfer: receiver */
5207 5208
				eaten = tcp_queue_rcv(sk, skb, tcp_header_len,
						      &fragstolen);
L
Linus Torvalds 已提交
5209 5210
			}

5211
			tcp_event_data_recv(sk, skb);
L
Linus Torvalds 已提交
5212 5213 5214 5215

			if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) {
				/* Well, only one small jumplet in fast path... */
				tcp_ack(sk, skb, FLAG_DATA);
5216
				tcp_data_snd_check(sk);
5217
				if (!inet_csk_ack_scheduled(sk))
L
Linus Torvalds 已提交
5218 5219 5220
					goto no_ack;
			}

D
Dan Williams 已提交
5221
			__tcp_ack_snd_check(sk, 0);
L
Linus Torvalds 已提交
5222 5223
no_ack:
			if (eaten)
5224
				kfree_skb_partial(skb, fragstolen);
5225
			sk->sk_data_ready(sk);
5226
			return;
L
Linus Torvalds 已提交
5227 5228 5229 5230
		}
	}

slow_path:
5231
	if (len < (th->doff << 2) || tcp_checksum_complete_user(sk, skb))
L
Linus Torvalds 已提交
5232 5233
		goto csum_error;

5234
	if (!th->ack && !th->rst && !th->syn)
5235 5236
		goto discard;

L
Linus Torvalds 已提交
5237 5238 5239 5240
	/*
	 *	Standard slow path.
	 */

E
Eric Dumazet 已提交
5241
	if (!tcp_validate_incoming(sk, skb, th, 1))
5242
		return;
L
Linus Torvalds 已提交
5243 5244

step5:
5245
	if (tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT) < 0)
5246
		goto discard;
L
Linus Torvalds 已提交
5247

5248
	tcp_rcv_rtt_measure_ts(sk, skb);
L
Linus Torvalds 已提交
5249 5250 5251 5252 5253 5254 5255

	/* Process urgent data. */
	tcp_urg(sk, skb, th);

	/* step 7: process the segment text */
	tcp_data_queue(sk, skb);

5256
	tcp_data_snd_check(sk);
L
Linus Torvalds 已提交
5257
	tcp_ack_snd_check(sk);
5258
	return;
L
Linus Torvalds 已提交
5259 5260

csum_error:
5261
	TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_CSUMERRORS);
5262
	TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS);
L
Linus Torvalds 已提交
5263 5264 5265 5266

discard:
	__kfree_skb(skb);
}
E
Eric Dumazet 已提交
5267
EXPORT_SYMBOL(tcp_rcv_established);
L
Linus Torvalds 已提交
5268

P
Pavel Emelyanov 已提交
5269 5270 5271 5272 5273 5274 5275
void tcp_finish_connect(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct inet_connection_sock *icsk = inet_csk(sk);

	tcp_set_state(sk, TCP_ESTABLISHED);

D
David S. Miller 已提交
5276
	if (skb != NULL) {
E
Eric Dumazet 已提交
5277
		icsk->icsk_af_ops->sk_rx_dst_set(sk, skb);
P
Pavel Emelyanov 已提交
5278
		security_inet_conn_established(sk, skb);
D
David S. Miller 已提交
5279
	}
P
Pavel Emelyanov 已提交
5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308

	/* Make sure socket is routed, for correct metrics.  */
	icsk->icsk_af_ops->rebuild_header(sk);

	tcp_init_metrics(sk);

	tcp_init_congestion_control(sk);

	/* Prevent spurious tcp_cwnd_restart() on first data
	 * packet.
	 */
	tp->lsndtime = tcp_time_stamp;

	tcp_init_buffer_space(sk);

	if (sock_flag(sk, SOCK_KEEPOPEN))
		inet_csk_reset_keepalive_timer(sk, keepalive_time_when(tp));

	if (!tp->rx_opt.snd_wscale)
		__tcp_fast_path_on(tp, tp->snd_wnd);
	else
		tp->pred_flags = 0;

	if (!sock_flag(sk, SOCK_DEAD)) {
		sk->sk_state_change(sk);
		sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
	}
}

5309 5310 5311 5312
static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack,
				    struct tcp_fastopen_cookie *cookie)
{
	struct tcp_sock *tp = tcp_sk(sk);
5313
	struct sk_buff *data = tp->syn_data ? tcp_write_queue_head(sk) : NULL;
5314
	u16 mss = tp->rx_opt.mss_clamp;
5315
	bool syn_drop;
5316 5317 5318 5319 5320 5321 5322

	if (mss == tp->rx_opt.user_mss) {
		struct tcp_options_received opt;

		/* Get original SYNACK MSS value if user MSS sets mss_clamp */
		tcp_clear_options(&opt);
		opt.user_mss = opt.mss_clamp = 0;
C
Christoph Paasch 已提交
5323
		tcp_parse_options(synack, &opt, 0, NULL);
5324 5325 5326
		mss = opt.mss_clamp;
	}

5327 5328 5329
	if (!tp->syn_fastopen)  /* Ignore an unsolicited cookie */
		cookie->len = -1;

5330 5331 5332 5333
	/* The SYN-ACK neither has cookie nor acknowledges the data. Presumably
	 * the remote receives only the retransmitted (regular) SYNs: either
	 * the original SYN-data or the corresponding SYN-ACK is lost.
	 */
5334
	syn_drop = (cookie->len <= 0 && data && tp->total_retrans);
5335 5336

	tcp_fastopen_cache_set(sk, mss, cookie, syn_drop);
5337 5338

	if (data) { /* Retransmit unacked data in SYN */
5339 5340 5341 5342 5343
		tcp_for_write_queue_from(data, sk) {
			if (data == tcp_send_head(sk) ||
			    __tcp_retransmit_skb(sk, data))
				break;
		}
5344
		tcp_rearm_rto(sk);
5345
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVEFAIL);
5346 5347
		return true;
	}
Y
Yuchung Cheng 已提交
5348
	tp->syn_data_acked = tp->syn_data;
5349 5350
	if (tp->syn_data_acked)
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
5351 5352 5353
	return false;
}

L
Linus Torvalds 已提交
5354
static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
5355
					 const struct tcphdr *th, unsigned int len)
L
Linus Torvalds 已提交
5356
{
5357
	struct inet_connection_sock *icsk = inet_csk(sk);
5358
	struct tcp_sock *tp = tcp_sk(sk);
5359
	struct tcp_fastopen_cookie foc = { .len = -1 };
5360
	int saved_clamp = tp->rx_opt.mss_clamp;
L
Linus Torvalds 已提交
5361

C
Christoph Paasch 已提交
5362
	tcp_parse_options(skb, &tp->rx_opt, 0, &foc);
5363
	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
5364
		tp->rx_opt.rcv_tsecr -= tp->tsoffset;
L
Linus Torvalds 已提交
5365 5366 5367 5368 5369 5370 5371 5372 5373 5374

	if (th->ack) {
		/* rfc793:
		 * "If the state is SYN-SENT then
		 *    first check the ACK bit
		 *      If the ACK bit is set
		 *	  If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send
		 *        a reset (unless the RST bit is set, if so drop
		 *        the segment and return)"
		 */
5375 5376
		if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) ||
		    after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt))
L
Linus Torvalds 已提交
5377 5378 5379 5380 5381
			goto reset_and_undo;

		if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
		    !between(tp->rx_opt.rcv_tsecr, tp->retrans_stamp,
			     tcp_time_stamp)) {
5382
			NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSACTIVEREJECTED);
L
Linus Torvalds 已提交
5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415
			goto reset_and_undo;
		}

		/* Now ACK is acceptable.
		 *
		 * "If the RST bit is set
		 *    If the ACK was acceptable then signal the user "error:
		 *    connection reset", drop the segment, enter CLOSED state,
		 *    delete TCB, and return."
		 */

		if (th->rst) {
			tcp_reset(sk);
			goto discard;
		}

		/* rfc793:
		 *   "fifth, if neither of the SYN or RST bits is set then
		 *    drop the segment and return."
		 *
		 *    See note below!
		 *                                        --ANK(990513)
		 */
		if (!th->syn)
			goto discard_and_undo;

		/* rfc793:
		 *   "If the SYN bit is on ...
		 *    are acceptable then ...
		 *    (our SYN has been ACKed), change the connection
		 *    state to ESTABLISHED..."
		 */

5416
		tcp_ecn_rcv_synack(tp, th);
L
Linus Torvalds 已提交
5417

5418
		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
L
Linus Torvalds 已提交
5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446
		tcp_ack(sk, skb, FLAG_SLOWPATH);

		/* Ok.. it's good. Set up sequence numbers and
		 * move to established.
		 */
		tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;

		/* RFC1323: The window in SYN & SYN/ACK segments is
		 * never scaled.
		 */
		tp->snd_wnd = ntohs(th->window);

		if (!tp->rx_opt.wscale_ok) {
			tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0;
			tp->window_clamp = min(tp->window_clamp, 65535U);
		}

		if (tp->rx_opt.saw_tstamp) {
			tp->rx_opt.tstamp_ok	   = 1;
			tp->tcp_header_len =
				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
			tp->advmss	    -= TCPOLEN_TSTAMP_ALIGNED;
			tcp_store_ts_recent(tp);
		} else {
			tp->tcp_header_len = sizeof(struct tcphdr);
		}

5447 5448
		if (tcp_is_sack(tp) && sysctl_tcp_fack)
			tcp_enable_fack(tp);
L
Linus Torvalds 已提交
5449

J
John Heffner 已提交
5450
		tcp_mtup_init(sk);
5451
		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
L
Linus Torvalds 已提交
5452 5453 5454 5455 5456 5457
		tcp_initialize_rcv_mss(sk);

		/* Remember, tcp_poll() does not lock socket!
		 * Change state from SYN-SENT only after copied_seq
		 * is initialized. */
		tp->copied_seq = tp->rcv_nxt;
5458

R
Ralf Baechle 已提交
5459
		smp_mb();
L
Linus Torvalds 已提交
5460

P
Pavel Emelyanov 已提交
5461
		tcp_finish_connect(sk, skb);
L
Linus Torvalds 已提交
5462

5463 5464
		if ((tp->syn_fastopen || tp->syn_data) &&
		    tcp_rcv_fastopen_synack(sk, skb, &foc))
5465 5466
			return -1;

5467 5468 5469
		if (sk->sk_write_pending ||
		    icsk->icsk_accept_queue.rskq_defer_accept ||
		    icsk->icsk_ack.pingpong) {
L
Linus Torvalds 已提交
5470 5471 5472 5473 5474 5475 5476
			/* Save one ACK. Data will be ready after
			 * several ticks, if write_pending is set.
			 *
			 * It may be deleted, but with this feature tcpdumps
			 * look so _wonderfully_ clever, that I was not able
			 * to stand against the temptation 8)     --ANK
			 */
5477
			inet_csk_schedule_ack(sk);
5478
			icsk->icsk_ack.lrcvtime = tcp_time_stamp;
5479
			tcp_enter_quickack_mode(sk);
5480 5481
			inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
						  TCP_DELACK_MAX, TCP_RTO_MAX);
L
Linus Torvalds 已提交
5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504

discard:
			__kfree_skb(skb);
			return 0;
		} else {
			tcp_send_ack(sk);
		}
		return -1;
	}

	/* No ACK in the segment */

	if (th->rst) {
		/* rfc793:
		 * "If the RST bit is set
		 *
		 *      Otherwise (no ACK) drop the segment and return."
		 */

		goto discard_and_undo;
	}

	/* PAWS check. */
5505
	if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp &&
I
Ilpo Järvinen 已提交
5506
	    tcp_paws_reject(&tp->rx_opt, 0))
L
Linus Torvalds 已提交
5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534
		goto discard_and_undo;

	if (th->syn) {
		/* We see SYN without ACK. It is attempt of
		 * simultaneous connect with crossed SYNs.
		 * Particularly, it can be connect to self.
		 */
		tcp_set_state(sk, TCP_SYN_RECV);

		if (tp->rx_opt.saw_tstamp) {
			tp->rx_opt.tstamp_ok = 1;
			tcp_store_ts_recent(tp);
			tp->tcp_header_len =
				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
		} else {
			tp->tcp_header_len = sizeof(struct tcphdr);
		}

		tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;

		/* RFC1323: The window in SYN & SYN/ACK segments is
		 * never scaled.
		 */
		tp->snd_wnd    = ntohs(th->window);
		tp->snd_wl1    = TCP_SKB_CB(skb)->seq;
		tp->max_window = tp->snd_wnd;

5535
		tcp_ecn_rcv_syn(tp, th);
L
Linus Torvalds 已提交
5536

J
John Heffner 已提交
5537
		tcp_mtup_init(sk);
5538
		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
L
Linus Torvalds 已提交
5539 5540 5541 5542 5543
		tcp_initialize_rcv_mss(sk);

		tcp_send_synack(sk);
#if 0
		/* Note, we could accept data and URG from this segment.
5544 5545 5546
		 * There are no obstacles to make this (except that we must
		 * either change tcp_recvmsg() to prevent it from returning data
		 * before 3WHS completes per RFC793, or employ TCP Fast Open).
L
Linus Torvalds 已提交
5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575
		 *
		 * However, if we ignore data in ACKless segments sometimes,
		 * we have no reasons to accept it sometimes.
		 * Also, seems the code doing it in step6 of tcp_rcv_state_process
		 * is not flawless. So, discard packet for sanity.
		 * Uncomment this return to process the data.
		 */
		return -1;
#else
		goto discard;
#endif
	}
	/* "fifth, if neither of the SYN or RST bits is set then
	 * drop the segment and return."
	 */

discard_and_undo:
	tcp_clear_options(&tp->rx_opt);
	tp->rx_opt.mss_clamp = saved_clamp;
	goto discard;

reset_and_undo:
	tcp_clear_options(&tp->rx_opt);
	tp->rx_opt.mss_clamp = saved_clamp;
	return 1;
}

/*
 *	This function implements the receiving procedure of RFC 793 for
5576
 *	all states except ESTABLISHED and TIME_WAIT.
L
Linus Torvalds 已提交
5577 5578 5579
 *	It's called from both tcp_v4_rcv and tcp_v6_rcv and should be
 *	address independent.
 */
5580

L
Linus Torvalds 已提交
5581
int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb,
5582
			  const struct tcphdr *th, unsigned int len)
L
Linus Torvalds 已提交
5583 5584
{
	struct tcp_sock *tp = tcp_sk(sk);
5585
	struct inet_connection_sock *icsk = inet_csk(sk);
5586
	struct request_sock *req;
L
Linus Torvalds 已提交
5587
	int queued = 0;
5588
	bool acceptable;
5589
	u32 synack_stamp;
L
Linus Torvalds 已提交
5590 5591 5592 5593 5594 5595 5596 5597

	tp->rx_opt.saw_tstamp = 0;

	switch (sk->sk_state) {
	case TCP_CLOSE:
		goto discard;

	case TCP_LISTEN:
S
Stephen Hemminger 已提交
5598
		if (th->ack)
L
Linus Torvalds 已提交
5599 5600
			return 1;

S
Stephen Hemminger 已提交
5601
		if (th->rst)
L
Linus Torvalds 已提交
5602 5603
			goto discard;

S
Stephen Hemminger 已提交
5604
		if (th->syn) {
E
Eric Dumazet 已提交
5605 5606
			if (th->fin)
				goto discard;
5607
			if (icsk->icsk_af_ops->conn_request(sk, skb) < 0)
L
Linus Torvalds 已提交
5608 5609
				return 1;

5610 5611
			/* Now we have several options: In theory there is
			 * nothing else in the frame. KA9Q has an option to
L
Linus Torvalds 已提交
5612
			 * send data with the syn, BSD accepts data with the
5613 5614 5615
			 * syn up to the [to be] advertised window and
			 * Solaris 2.1 gives you a protocol error. For now
			 * we just ignore it, that fits the spec precisely
L
Linus Torvalds 已提交
5616 5617 5618
			 * and avoids incompatibilities. It would be nice in
			 * future to drop through and process the data.
			 *
5619
			 * Now that TTCP is starting to be used we ought to
L
Linus Torvalds 已提交
5620 5621
			 * queue this data.
			 * But, this leaves one open to an easy denial of
5622
			 * service attack, and SYN cookies can't defend
L
Linus Torvalds 已提交
5623
			 * against this problem. So, we drop the data
5624 5625
			 * in the interest of security over speed unless
			 * it's still in use.
L
Linus Torvalds 已提交
5626
			 */
5627 5628
			kfree_skb(skb);
			return 0;
L
Linus Torvalds 已提交
5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639
		}
		goto discard;

	case TCP_SYN_SENT:
		queued = tcp_rcv_synsent_state_process(sk, skb, th, len);
		if (queued >= 0)
			return queued;

		/* Do step6 onward by hand. */
		tcp_urg(sk, skb, th);
		__kfree_skb(skb);
5640
		tcp_data_snd_check(sk);
L
Linus Torvalds 已提交
5641 5642 5643
		return 0;
	}

5644 5645
	req = tp->fastopen_rsk;
	if (req != NULL) {
5646
		WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV &&
5647 5648 5649 5650
		    sk->sk_state != TCP_FIN_WAIT1);

		if (tcp_check_req(sk, skb, req, NULL, true) == NULL)
			goto discard;
5651
	}
5652

5653
	if (!th->ack && !th->rst && !th->syn)
5654 5655
		goto discard;

5656
	if (!tcp_validate_incoming(sk, skb, th, 0))
E
Eric Dumazet 已提交
5657
		return 0;
L
Linus Torvalds 已提交
5658 5659

	/* step 5: check the ACK field */
5660 5661
	acceptable = tcp_ack(sk, skb, FLAG_SLOWPATH |
				      FLAG_UPDATE_TS_RECENT) > 0;
5662

5663 5664
	switch (sk->sk_state) {
	case TCP_SYN_RECV:
5665 5666
		if (!acceptable)
			return 1;
L
Linus Torvalds 已提交
5667

5668 5669 5670 5671
		/* Once we leave TCP_SYN_RECV, we no longer need req
		 * so release it.
		 */
		if (req) {
5672
			synack_stamp = tcp_rsk(req)->snt_synack;
5673 5674 5675
			tp->total_retrans = req->num_retrans;
			reqsk_fastopen_remove(sk, req, false);
		} else {
5676
			synack_stamp = tp->lsndtime;
5677 5678 5679 5680 5681 5682
			/* Make sure socket is routed, for correct metrics. */
			icsk->icsk_af_ops->rebuild_header(sk);
			tcp_init_congestion_control(sk);

			tcp_mtup_init(sk);
			tp->copied_seq = tp->rcv_nxt;
E
Eric Dumazet 已提交
5683
			tcp_init_buffer_space(sk);
5684 5685 5686 5687
		}
		smp_mb();
		tcp_set_state(sk, TCP_ESTABLISHED);
		sk->sk_state_change(sk);
L
Linus Torvalds 已提交
5688

5689 5690 5691 5692 5693 5694 5695 5696 5697 5698
		/* Note, that this wakeup is only for marginal crossed SYN case.
		 * Passively open sockets are not waked up, because
		 * sk->sk_sleep == NULL and sk->sk_socket == NULL.
		 */
		if (sk->sk_socket)
			sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);

		tp->snd_una = TCP_SKB_CB(skb)->ack_seq;
		tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
5699
		tcp_synack_rtt_meas(sk, synack_stamp);
5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711

		if (tp->rx_opt.tstamp_ok)
			tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;

		if (req) {
			/* Re-arm the timer because data may have been sent out.
			 * This is similar to the regular data transmission case
			 * when new data has just been ack'ed.
			 *
			 * (TFO) - we could try to be more aggressive and
			 * retransmitting any data sooner based on when they
			 * are sent out.
5712
			 */
5713 5714 5715
			tcp_rearm_rto(sk);
		} else
			tcp_init_metrics(sk);
L
Linus Torvalds 已提交
5716

5717 5718
		tcp_update_pacing_rate(sk);

5719 5720 5721 5722 5723
		/* Prevent spurious tcp_cwnd_restart() on first data packet */
		tp->lsndtime = tcp_time_stamp;

		tcp_initialize_rcv_mss(sk);
		tcp_fast_path_on(tp);
5724 5725
		break;

5726 5727 5728 5729
	case TCP_FIN_WAIT1: {
		struct dst_entry *dst;
		int tmo;

5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740
		/* If we enter the TCP_FIN_WAIT1 state and we are a
		 * Fast Open socket and this is the first acceptable
		 * ACK we have received, this would have acknowledged
		 * our SYNACK so stop the SYNACK timer.
		 */
		if (req != NULL) {
			/* Return RST if ack_seq is invalid.
			 * Note that RFC793 only says to generate a
			 * DUPACK for it but for TCP Fast Open it seems
			 * better to treat this case like TCP_SYN_RECV
			 * above.
5741
			 */
5742 5743 5744 5745 5746 5747
			if (!acceptable)
				return 1;
			/* We no longer need the request sock. */
			reqsk_fastopen_remove(sk, req, false);
			tcp_rearm_rto(sk);
		}
5748 5749
		if (tp->snd_una != tp->write_seq)
			break;
5750

5751 5752
		tcp_set_state(sk, TCP_FIN_WAIT2);
		sk->sk_shutdown |= SEND_SHUTDOWN;
5753

5754 5755 5756
		dst = __sk_dst_get(sk);
		if (dst)
			dst_confirm(dst);
5757

5758 5759 5760 5761 5762
		if (!sock_flag(sk, SOCK_DEAD)) {
			/* Wake up lingering close() */
			sk->sk_state_change(sk);
			break;
		}
L
Linus Torvalds 已提交
5763

5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785
		if (tp->linger2 < 0 ||
		    (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
		     after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt))) {
			tcp_done(sk);
			NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
			return 1;
		}

		tmo = tcp_fin_time(sk);
		if (tmo > TCP_TIMEWAIT_LEN) {
			inet_csk_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN);
		} else if (th->fin || sock_owned_by_user(sk)) {
			/* Bad case. We could lose such FIN otherwise.
			 * It is not a big problem, but it looks confusing
			 * and not so rare event. We still can lose it now,
			 * if it spins in bh_lock_sock(), but it is really
			 * marginal case.
			 */
			inet_csk_reset_keepalive_timer(sk, tmo);
		} else {
			tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
			goto discard;
5786 5787
		}
		break;
5788
	}
L
Linus Torvalds 已提交
5789

5790 5791 5792 5793 5794 5795
	case TCP_CLOSING:
		if (tp->snd_una == tp->write_seq) {
			tcp_time_wait(sk, TCP_TIME_WAIT, 0);
			goto discard;
		}
		break;
L
Linus Torvalds 已提交
5796

5797 5798 5799 5800 5801
	case TCP_LAST_ACK:
		if (tp->snd_una == tp->write_seq) {
			tcp_update_metrics(sk);
			tcp_done(sk);
			goto discard;
L
Linus Torvalds 已提交
5802
		}
5803
		break;
5804
	}
L
Linus Torvalds 已提交
5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818

	/* step 6: check the URG bit */
	tcp_urg(sk, skb, th);

	/* step 7: process the segment text */
	switch (sk->sk_state) {
	case TCP_CLOSE_WAIT:
	case TCP_CLOSING:
	case TCP_LAST_ACK:
		if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
			break;
	case TCP_FIN_WAIT1:
	case TCP_FIN_WAIT2:
		/* RFC 793 says to queue data in these states,
5819
		 * RFC 1122 says we MUST send a reset.
L
Linus Torvalds 已提交
5820 5821 5822 5823 5824
		 * BSD 4.4 also does reset.
		 */
		if (sk->sk_shutdown & RCV_SHUTDOWN) {
			if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
			    after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
5825
				NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
L
Linus Torvalds 已提交
5826 5827 5828 5829 5830
				tcp_reset(sk);
				return 1;
			}
		}
		/* Fall through */
5831
	case TCP_ESTABLISHED:
L
Linus Torvalds 已提交
5832 5833 5834 5835 5836 5837 5838
		tcp_data_queue(sk, skb);
		queued = 1;
		break;
	}

	/* tcp_data could move socket to TIME-WAIT */
	if (sk->sk_state != TCP_CLOSE) {
5839
		tcp_data_snd_check(sk);
L
Linus Torvalds 已提交
5840 5841 5842
		tcp_ack_snd_check(sk);
	}

5843
	if (!queued) {
L
Linus Torvalds 已提交
5844 5845 5846 5847 5848 5849
discard:
		__kfree_skb(skb);
	}
	return 0;
}
EXPORT_SYMBOL(tcp_rcv_state_process);
O
Octavian Purdila 已提交
5850 5851 5852 5853 5854 5855

static inline void pr_drop_req(struct request_sock *req, __u16 port, int family)
{
	struct inet_request_sock *ireq = inet_rsk(req);

	if (family == AF_INET)
5856 5857
		net_dbg_ratelimited("drop open request from %pI4/%u\n",
				    &ireq->ir_rmt_addr, port);
5858 5859
#if IS_ENABLED(CONFIG_IPV6)
	else if (family == AF_INET6)
5860 5861
		net_dbg_ratelimited("drop open request from %pI6/%u\n",
				    &ireq->ir_v6_rmt_addr, port);
5862
#endif
O
Octavian Purdila 已提交
5863 5864
}

5865 5866 5867 5868 5869
/* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set
 *
 * If we receive a SYN packet with these bits set, it means a
 * network is playing bad games with TOS bits. In order to
 * avoid possible false congestion notifications, we disable
S
stephen hemminger 已提交
5870
 * TCP ECN negotiation.
5871 5872 5873 5874 5875 5876 5877 5878 5879
 *
 * Exception: tcp_ca wants ECN. This is required for DCTCP
 * congestion control; it requires setting ECT on all packets,
 * including SYN. We inverse the test in this case: If our
 * local socket wants ECN, but peer only set ece/cwr (but not
 * ECT in IP header) its probably a non-DCTCP aware sender.
 */
static void tcp_ecn_create_request(struct request_sock *req,
				   const struct sk_buff *skb,
5880 5881
				   const struct sock *listen_sk,
				   const struct dst_entry *dst)
5882 5883 5884 5885
{
	const struct tcphdr *th = tcp_hdr(skb);
	const struct net *net = sock_net(listen_sk);
	bool th_ecn = th->ece && th->cwr;
5886
	bool ect, need_ecn, ecn_ok;
5887 5888 5889 5890 5891 5892

	if (!th_ecn)
		return;

	ect = !INET_ECN_is_not_ect(TCP_SKB_CB(skb)->ip_dsfield);
	need_ecn = tcp_ca_needs_ecn(listen_sk);
5893
	ecn_ok = net->ipv4.sysctl_tcp_ecn || dst_feature(dst, RTAX_FEATURE_ECN);
5894

5895
	if (!ect && !need_ecn && ecn_ok)
5896 5897 5898 5899 5900
		inet_rsk(req)->ecn_ok = 1;
	else if (ect && need_ecn)
		inet_rsk(req)->ecn_ok = 1;
}

O
Octavian Purdila 已提交
5901 5902 5903 5904 5905 5906 5907 5908
int tcp_conn_request(struct request_sock_ops *rsk_ops,
		     const struct tcp_request_sock_ops *af_ops,
		     struct sock *sk, struct sk_buff *skb)
{
	struct tcp_options_received tmp_opt;
	struct request_sock *req;
	struct tcp_sock *tp = tcp_sk(sk);
	struct dst_entry *dst = NULL;
5909
	__u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
O
Octavian Purdila 已提交
5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959
	bool want_cookie = false, fastopen;
	struct flowi fl;
	struct tcp_fastopen_cookie foc = { .len = -1 };
	int err;


	/* TW buckets are converted to open requests without
	 * limitations, they conserve resources and peer is
	 * evidently real one.
	 */
	if ((sysctl_tcp_syncookies == 2 ||
	     inet_csk_reqsk_queue_is_full(sk)) && !isn) {
		want_cookie = tcp_syn_flood_action(sk, skb, rsk_ops->slab_name);
		if (!want_cookie)
			goto drop;
	}


	/* Accept backlog is full. If we have already queued enough
	 * of warm entries in syn queue, drop request. It is better than
	 * clogging syn queue with openreqs with exponentially increasing
	 * timeout.
	 */
	if (sk_acceptq_is_full(sk) && inet_csk_reqsk_queue_young(sk) > 1) {
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
		goto drop;
	}

	req = inet_reqsk_alloc(rsk_ops);
	if (!req)
		goto drop;

	tcp_rsk(req)->af_specific = af_ops;

	tcp_clear_options(&tmp_opt);
	tmp_opt.mss_clamp = af_ops->mss_clamp;
	tmp_opt.user_mss  = tp->rx_opt.user_mss;
	tcp_parse_options(skb, &tmp_opt, 0, want_cookie ? NULL : &foc);

	if (want_cookie && !tmp_opt.saw_tstamp)
		tcp_clear_options(&tmp_opt);

	tmp_opt.tstamp_ok = tmp_opt.saw_tstamp;
	tcp_openreq_init(req, &tmp_opt, skb, sk);

	af_ops->init_req(req, sk, skb);

	if (security_inet_conn_request(sk, skb, req))
		goto drop_and_free;

5960
	if (!want_cookie && !isn) {
O
Octavian Purdila 已提交
5961 5962 5963 5964 5965 5966 5967 5968 5969
		/* VJ's idea. We save last timestamp seen
		 * from the destination in peer table, when entering
		 * state TIME-WAIT, and check against it before
		 * accepting new connection request.
		 *
		 * If "isn" is not zero, this request hit alive
		 * timewait bucket, so that all the necessary checks
		 * are made in the function processing timewait state.
		 */
5970
		if (tcp_death_row.sysctl_tw_recycle) {
O
Octavian Purdila 已提交
5971 5972 5973
			bool strict;

			dst = af_ops->route_req(sk, &fl, req, &strict);
5974

O
Octavian Purdila 已提交
5975
			if (dst && strict &&
5976 5977
			    !tcp_peer_is_proven(req, dst, true,
						tmp_opt.saw_tstamp)) {
O
Octavian Purdila 已提交
5978 5979 5980 5981 5982 5983 5984 5985
				NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_PAWSPASSIVEREJECTED);
				goto drop_and_release;
			}
		}
		/* Kill the following clause, if you dislike this way. */
		else if (!sysctl_tcp_syncookies &&
			 (sysctl_max_syn_backlog - inet_csk_reqsk_queue_len(sk) <
			  (sysctl_max_syn_backlog >> 2)) &&
5986 5987
			 !tcp_peer_is_proven(req, dst, false,
					     tmp_opt.saw_tstamp)) {
O
Octavian Purdila 已提交
5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007
			/* Without syncookies last quarter of
			 * backlog is filled with destinations,
			 * proven to be alive.
			 * It means that we continue to communicate
			 * to destinations, already remembered
			 * to the moment of synflood.
			 */
			pr_drop_req(req, ntohs(tcp_hdr(skb)->source),
				    rsk_ops->family);
			goto drop_and_release;
		}

		isn = af_ops->init_seq(skb);
	}
	if (!dst) {
		dst = af_ops->route_req(sk, &fl, req, NULL);
		if (!dst)
			goto drop_and_free;
	}

6008 6009 6010 6011 6012 6013 6014 6015 6016
	tcp_ecn_create_request(req, skb, sk, dst);

	if (want_cookie) {
		isn = cookie_init_sequence(af_ops, sk, skb, &req->mss);
		req->cookie_ts = tmp_opt.tstamp_ok;
		if (!tmp_opt.tstamp_ok)
			inet_rsk(req)->ecn_ok = 0;
	}

O
Octavian Purdila 已提交
6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041
	tcp_rsk(req)->snt_isn = isn;
	tcp_openreq_init_rwin(req, sk, dst);
	fastopen = !want_cookie &&
		   tcp_try_fastopen(sk, skb, req, &foc, dst);
	err = af_ops->send_synack(sk, dst, &fl, req,
				  skb_get_queue_mapping(skb), &foc);
	if (!fastopen) {
		if (err || want_cookie)
			goto drop_and_free;

		tcp_rsk(req)->listener = NULL;
		af_ops->queue_hash_add(sk, req, TCP_TIMEOUT_INIT);
	}

	return 0;

drop_and_release:
	dst_release(dst);
drop_and_free:
	reqsk_free(req);
drop:
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS);
	return 0;
}
EXPORT_SYMBOL(tcp_conn_request);