tcp_input.c 183.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
L
Linus Torvalds 已提交
2 3 4 5 6 7 8
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		Implementation of the Transmission Control Protocol(TCP).
 *
9
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Mark Evans, <evansmp@uhura.aston.ac.uk>
 *		Corey Minyard <wf-rch!minyard@relay.EU.net>
 *		Florian La Roche, <flla@stud.uni-sb.de>
 *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
 *		Linus Torvalds, <torvalds@cs.helsinki.fi>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Matthew Dillon, <dillon@apollo.west.oic.com>
 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *		Jorge Cwik, <jorge@laser.satlink.net>
 */

/*
 * Changes:
 *		Pedro Roque	:	Fast Retransmit/Recovery.
 *					Two receive queues.
 *					Retransmit queue handled by TCP.
 *					Better retransmit timer handling.
 *					New congestion avoidance.
 *					Header prediction.
 *					Variable renaming.
 *
 *		Eric		:	Fast Retransmit.
 *		Randy Scott	:	MSS option defines.
 *		Eric Schenk	:	Fixes to slow start algorithm.
 *		Eric Schenk	:	Yet another double ACK bug.
 *		Eric Schenk	:	Delayed ACK bug fixes.
 *		Eric Schenk	:	Floyd style fast retrans war avoidance.
 *		David S. Miller	:	Don't allow zero congestion window.
 *		Eric Schenk	:	Fix retransmitter so that it sends
 *					next packet on ack of previous packet.
 *		Andi Kleen	:	Moved open_request checking here
 *					and process RSTs for open_requests.
 *		Andi Kleen	:	Better prune_queue, and other fixes.
S
Stephen Hemminger 已提交
44
 *		Andrey Savochkin:	Fix RTT measurements in the presence of
L
Linus Torvalds 已提交
45 46 47 48 49 50 51
 *					timestamps.
 *		Andrey Savochkin:	Check sequence numbers correctly when
 *					removing SACKs due to in sequence incoming
 *					data segments.
 *		Andi Kleen:		Make sure we never ack data there is not
 *					enough room for. Also make this condition
 *					a fatal error if it might still happen.
52
 *		Andi Kleen:		Add tcp_measure_rcv_mss to make
L
Linus Torvalds 已提交
53
 *					connections with MSS<min(MTU,ann. MSS)
54
 *					work without delayed acks.
L
Linus Torvalds 已提交
55 56 57 58 59 60 61 62 63 64
 *		Andi Kleen:		Process packets with PSH set in the
 *					fast path.
 *		J Hadi Salim:		ECN support
 *	 	Andrei Gurtov,
 *		Pasi Sarolahti,
 *		Panu Kuhlberg:		Experimental audit of TCP (re)transmission
 *					engine. Lots of bugs are found.
 *		Pasi Sarolahti:		F-RTO for dealing with spurious RTOs
 */

65 66
#define pr_fmt(fmt) "TCP: " fmt

L
Linus Torvalds 已提交
67
#include <linux/mm.h>
68
#include <linux/slab.h>
L
Linus Torvalds 已提交
69 70
#include <linux/module.h>
#include <linux/sysctl.h>
71
#include <linux/kernel.h>
72
#include <linux/prefetch.h>
73
#include <net/dst.h>
L
Linus Torvalds 已提交
74 75 76 77
#include <net/tcp.h>
#include <net/inet_common.h>
#include <linux/ipsec.h>
#include <asm/unaligned.h>
78
#include <linux/errqueue.h>
79
#include <trace/events/tcp.h>
80
#include <linux/static_key.h>
L
Linus Torvalds 已提交
81

82
int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
L
Linus Torvalds 已提交
83 84 85 86 87 88 89 90

#define FLAG_DATA		0x01 /* Incoming frame contained data.		*/
#define FLAG_WIN_UPDATE		0x02 /* Incoming ACK was a window update.	*/
#define FLAG_DATA_ACKED		0x04 /* This ACK acknowledged new data.		*/
#define FLAG_RETRANS_DATA_ACKED	0x08 /* "" "" some of which was retransmitted.	*/
#define FLAG_SYN_ACKED		0x10 /* This ACK acknowledged SYN.		*/
#define FLAG_DATA_SACKED	0x20 /* New SACK.				*/
#define FLAG_ECE		0x40 /* ECE in this ACK				*/
91
#define FLAG_LOST_RETRANS	0x80 /* This ACK marks some retransmission lost */
92
#define FLAG_SLOWPATH		0x100 /* Do not skip RFC checks for window update.*/
Y
Yuchung Cheng 已提交
93
#define FLAG_ORIG_SACK_ACKED	0x200 /* Never retransmitted data are (s)acked	*/
94
#define FLAG_SND_UNA_ADVANCED	0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
95
#define FLAG_DSACKING_ACK	0x800 /* SACK blocks contained D-SACK info */
96
#define FLAG_SET_XMIT_TIMER	0x1000 /* Set TLP or RTO timer */
97
#define FLAG_SACK_RENEGING	0x2000 /* snd_una advanced to a sacked seq */
98
#define FLAG_UPDATE_TS_RECENT	0x4000 /* tcp_replace_ts_recent() */
99
#define FLAG_NO_CHALLENGE_ACK	0x8000 /* do not call tcp_send_challenge_ack()	*/
100
#define FLAG_ACK_MAYBE_DELAYED	0x10000 /* Likely a delayed ACK */
L
Linus Torvalds 已提交
101 102 103

#define FLAG_ACKED		(FLAG_DATA_ACKED|FLAG_SYN_ACKED)
#define FLAG_NOT_DUP		(FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED)
104
#define FLAG_CA_ALERT		(FLAG_DATA_SACKED|FLAG_ECE|FLAG_DSACKING_ACK)
L
Linus Torvalds 已提交
105 106 107
#define FLAG_FORWARD_PROGRESS	(FLAG_ACKED|FLAG_DATA_SACKED)

#define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH)
108
#define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH))
L
Linus Torvalds 已提交
109

110 111 112 113
#define REXMIT_NONE	0 /* no loss recovery to do */
#define REXMIT_LOST	1 /* retransmit packets marked lost */
#define REXMIT_NEW	2 /* FRTO-style transmit of unsent/new packets */

I
Ilya Lesokhin 已提交
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
#if IS_ENABLED(CONFIG_TLS_DEVICE)
static DEFINE_STATIC_KEY_FALSE(clean_acked_data_enabled);

void clean_acked_data_enable(struct inet_connection_sock *icsk,
			     void (*cad)(struct sock *sk, u32 ack_seq))
{
	icsk->icsk_clean_acked = cad;
	static_branch_inc(&clean_acked_data_enabled);
}
EXPORT_SYMBOL_GPL(clean_acked_data_enable);

void clean_acked_data_disable(struct inet_connection_sock *icsk)
{
	static_branch_dec(&clean_acked_data_enabled);
	icsk->icsk_clean_acked = NULL;
}
EXPORT_SYMBOL_GPL(clean_acked_data_disable);
#endif

133 134
static void tcp_gro_dev_warn(struct sock *sk, const struct sk_buff *skb,
			     unsigned int len)
135 136 137 138 139 140 141 142 143 144
{
	static bool __once __read_mostly;

	if (!__once) {
		struct net_device *dev;

		__once = true;

		rcu_read_lock();
		dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
145 146 147
		if (!dev || len >= dev->mtu)
			pr_warn("%s: Driver has suspect GRO implementation, TCP performance may be compromised.\n",
				dev ? dev->name : "Unknown driver");
148 149 150 151
		rcu_read_unlock();
	}
}

152
/* Adapt the MSS value used to make delayed ack decision to the
L
Linus Torvalds 已提交
153
 * real world.
154
 */
155
static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
156
{
157
	struct inet_connection_sock *icsk = inet_csk(sk);
158
	const unsigned int lss = icsk->icsk_ack.last_seg_size;
159
	unsigned int len;
L
Linus Torvalds 已提交
160

161
	icsk->icsk_ack.last_seg_size = 0;
L
Linus Torvalds 已提交
162 163 164 165

	/* skb->len may jitter because of SACKs, even if peer
	 * sends good full-sized frames.
	 */
166
	len = skb_shinfo(skb)->gso_size ? : skb->len;
167
	if (len >= icsk->icsk_ack.rcv_mss) {
168 169
		icsk->icsk_ack.rcv_mss = min_t(unsigned int, len,
					       tcp_sk(sk)->advmss);
170 171 172 173
		/* Account for possibly-removed options */
		if (unlikely(len > icsk->icsk_ack.rcv_mss +
				   MAX_TCP_OPTION_SPACE))
			tcp_gro_dev_warn(sk, skb, len);
L
Linus Torvalds 已提交
174 175 176 177 178 179
	} else {
		/* Otherwise, we make more careful check taking into account,
		 * that SACKs block is variable.
		 *
		 * "len" is invariant segment length, including TCP header.
		 */
180
		len += skb->data - skb_transport_header(skb);
181
		if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) ||
L
Linus Torvalds 已提交
182 183 184 185 186 187
		    /* If PSH is not set, packet should be
		     * full sized, provided peer TCP is not badly broken.
		     * This observation (if it is correct 8)) allows
		     * to handle super-low mtu links fairly.
		     */
		    (len >= TCP_MIN_MSS + sizeof(struct tcphdr) &&
188
		     !(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) {
L
Linus Torvalds 已提交
189 190 191 192
			/* Subtract also invariant (if peer is RFC compliant),
			 * tcp header plus fixed timestamp option length.
			 * Resulting "len" is MSS free of SACK jitter.
			 */
193 194
			len -= tcp_sk(sk)->tcp_header_len;
			icsk->icsk_ack.last_seg_size = len;
L
Linus Torvalds 已提交
195
			if (len == lss) {
196
				icsk->icsk_ack.rcv_mss = len;
L
Linus Torvalds 已提交
197 198 199
				return;
			}
		}
200 201
		if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)
			icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2;
202
		icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
L
Linus Torvalds 已提交
203 204 205
	}
}

206
static void tcp_incr_quickack(struct sock *sk, unsigned int max_quickacks)
L
Linus Torvalds 已提交
207
{
208
	struct inet_connection_sock *icsk = inet_csk(sk);
209
	unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss);
L
Linus Torvalds 已提交
210

211 212
	if (quickacks == 0)
		quickacks = 2;
213
	quickacks = min(quickacks, max_quickacks);
214
	if (quickacks > icsk->icsk_ack.quick)
215
		icsk->icsk_ack.quick = quickacks;
L
Linus Torvalds 已提交
216 217
}

218
static void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks)
L
Linus Torvalds 已提交
219
{
220
	struct inet_connection_sock *icsk = inet_csk(sk);
221 222

	tcp_incr_quickack(sk, max_quickacks);
223 224
	icsk->icsk_ack.pingpong = 0;
	icsk->icsk_ack.ato = TCP_ATO_MIN;
L
Linus Torvalds 已提交
225 226 227 228 229 230
}

/* Send ACKs quickly, if "quick" count is not exhausted
 * and the session is not interactive.
 */

231
static bool tcp_in_quickack_mode(struct sock *sk)
L
Linus Torvalds 已提交
232
{
233
	const struct inet_connection_sock *icsk = inet_csk(sk);
234
	const struct dst_entry *dst = __sk_dst_get(sk);
E
Eric Dumazet 已提交
235

236 237
	return (dst && dst_metric(dst, RTAX_QUICKACK)) ||
		(icsk->icsk_ack.quick && !icsk->icsk_ack.pingpong);
L
Linus Torvalds 已提交
238 239
}

240
static void tcp_ecn_queue_cwr(struct tcp_sock *tp)
241
{
242
	if (tp->ecn_flags & TCP_ECN_OK)
243 244 245
		tp->ecn_flags |= TCP_ECN_QUEUE_CWR;
}

246
static void tcp_ecn_accept_cwr(struct tcp_sock *tp, const struct sk_buff *skb)
247 248 249 250 251
{
	if (tcp_hdr(skb)->cwr)
		tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
}

252
static void tcp_ecn_withdraw_cwr(struct tcp_sock *tp)
253 254 255 256
{
	tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
}

257
static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
258
{
259 260
	struct tcp_sock *tp = tcp_sk(sk);

261
	switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) {
262
	case INET_ECN_NOT_ECT:
263
		/* Funny extension: if ECT is not set on a segment,
264 265 266 267
		 * and we already seen ECT on a previous segment,
		 * it is probably a retransmit.
		 */
		if (tp->ecn_flags & TCP_ECN_SEEN)
268
			tcp_enter_quickack_mode(sk, 1);
269 270
		break;
	case INET_ECN_CE:
271 272
		if (tcp_ca_needs_ecn(sk))
			tcp_ca_event(sk, CA_EVENT_ECN_IS_CE);
273

274 275
		if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) {
			/* Better not delay acks, sender can have a very low cwnd */
276
			tcp_enter_quickack_mode(sk, 1);
277 278
			tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
		}
279 280
		tp->ecn_flags |= TCP_ECN_SEEN;
		break;
281
	default:
282 283
		if (tcp_ca_needs_ecn(sk))
			tcp_ca_event(sk, CA_EVENT_ECN_NO_CE);
284
		tp->ecn_flags |= TCP_ECN_SEEN;
285
		break;
286 287 288
	}
}

289
static void tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
290
{
291 292
	if (tcp_sk(sk)->ecn_flags & TCP_ECN_OK)
		__tcp_ecn_check_ce(sk, skb);
293 294 295
}

static void tcp_ecn_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th)
296
{
297
	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || th->cwr))
298 299 300
		tp->ecn_flags &= ~TCP_ECN_OK;
}

301
static void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th)
302
{
303
	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || !th->cwr))
304 305 306
		tp->ecn_flags &= ~TCP_ECN_OK;
}

307
static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr *th)
308
{
309
	if (th->ece && !th->syn && (tp->ecn_flags & TCP_ECN_OK))
E
Eric Dumazet 已提交
310 311
		return true;
	return false;
312 313
}

L
Linus Torvalds 已提交
314 315 316 317 318
/* Buffer size and advertised window tuning.
 *
 * 1. Tuning sk->sk_sndbuf, when connection enters established state.
 */

E
Eric Dumazet 已提交
319
static void tcp_sndbuf_expand(struct sock *sk)
L
Linus Torvalds 已提交
320
{
E
Eric Dumazet 已提交
321
	const struct tcp_sock *tp = tcp_sk(sk);
322
	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
E
Eric Dumazet 已提交
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
	int sndmem, per_mss;
	u32 nr_segs;

	/* Worst case is non GSO/TSO : each frame consumes one skb
	 * and skb->head is kmalloced using power of two area of memory
	 */
	per_mss = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) +
		  MAX_TCP_HEADER +
		  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

	per_mss = roundup_pow_of_two(per_mss) +
		  SKB_DATA_ALIGN(sizeof(struct sk_buff));

	nr_segs = max_t(u32, TCP_INIT_CWND, tp->snd_cwnd);
	nr_segs = max_t(u32, nr_segs, tp->reordering + 1);

	/* Fast Recovery (RFC 5681 3.2) :
	 * Cubic needs 1.7 factor, rounded to 2 to include
341
	 * extra cushion (application might react slowly to EPOLLOUT)
E
Eric Dumazet 已提交
342
	 */
343 344
	sndmem = ca_ops->sndbuf_expand ? ca_ops->sndbuf_expand(sk) : 2;
	sndmem *= nr_segs * per_mss;
L
Linus Torvalds 已提交
345

346
	if (sk->sk_sndbuf < sndmem)
347
		sk->sk_sndbuf = min(sndmem, sock_net(sk)->ipv4.sysctl_tcp_wmem[2]);
L
Linus Torvalds 已提交
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
}

/* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
 *
 * All tcp_full_space() is split to two parts: "network" buffer, allocated
 * forward and advertised in receiver window (tp->rcv_wnd) and
 * "application buffer", required to isolate scheduling/application
 * latencies from network.
 * window_clamp is maximal advertised window. It can be less than
 * tcp_full_space(), in this case tcp_full_space() - window_clamp
 * is reserved for "application" buffer. The less window_clamp is
 * the smoother our behaviour from viewpoint of network, but the lower
 * throughput and the higher sensitivity of the connection to losses. 8)
 *
 * rcv_ssthresh is more strict window_clamp used at "slow start"
 * phase to predict further behaviour of this connection.
 * It is used for two goals:
 * - to enforce header prediction at sender, even when application
 *   requires some significant "application buffer". It is check #1.
 * - to prevent pruning of receive queue because of misprediction
 *   of receiver window. Check #2.
 *
 * The scheme does not work when sender sends good segments opening
S
Stephen Hemminger 已提交
371
 * window and then starts to feed us spaghetti. But it should work
L
Linus Torvalds 已提交
372 373 374 375
 * in common situations. Otherwise, we have to rely on queue collapsing.
 */

/* Slow part of check#2. */
376
static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
377
{
378
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
379
	/* Optimize this! */
380
	int truesize = tcp_win_from_space(sk, skb->truesize) >> 1;
381
	int window = tcp_win_from_space(sk, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
L
Linus Torvalds 已提交
382 383 384

	while (tp->rcv_ssthresh <= window) {
		if (truesize <= skb->len)
385
			return 2 * inet_csk(sk)->icsk_ack.rcv_mss;
L
Linus Torvalds 已提交
386 387 388 389 390 391 392

		truesize >>= 1;
		window >>= 1;
	}
	return 0;
}

393
static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
394
{
395 396
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
397 398 399
	/* Check #1 */
	if (tp->rcv_ssthresh < tp->window_clamp &&
	    (int)tp->rcv_ssthresh < tcp_space(sk) &&
400
	    !tcp_under_memory_pressure(sk)) {
L
Linus Torvalds 已提交
401 402 403 404 405
		int incr;

		/* Check #2. Increase window, if skb with such overhead
		 * will fit to rcvbuf in future.
		 */
406
		if (tcp_win_from_space(sk, skb->truesize) <= skb->len)
407
			incr = 2 * tp->advmss;
L
Linus Torvalds 已提交
408
		else
409
			incr = __tcp_grow_window(sk, skb);
L
Linus Torvalds 已提交
410 411

		if (incr) {
412
			incr = max_t(int, incr, 2 * skb->len);
413 414
			tp->rcv_ssthresh = min(tp->rcv_ssthresh + incr,
					       tp->window_clamp);
415
			inet_csk(sk)->icsk_ack.quick |= 1;
L
Linus Torvalds 已提交
416 417 418 419 420 421 422
		}
	}
}

/* 3. Tuning rcvbuf, when connection enters established state. */
static void tcp_fixup_rcvbuf(struct sock *sk)
{
423 424
	u32 mss = tcp_sk(sk)->advmss;
	int rcvmem;
L
Linus Torvalds 已提交
425

426 427
	rcvmem = 2 * SKB_TRUESIZE(mss + MAX_TCP_HEADER) *
		 tcp_default_init_rwnd(mss);
428

E
Eric Dumazet 已提交
429 430 431
	/* Dynamic Right Sizing (DRS) has 2 to 3 RTT latency
	 * Allow enough cushion so that sender is not limited by our window
	 */
432
	if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf)
E
Eric Dumazet 已提交
433 434
		rcvmem <<= 2;

435
	if (sk->sk_rcvbuf < rcvmem)
436
		sk->sk_rcvbuf = min(rcvmem, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
L
Linus Torvalds 已提交
437 438
}

S
Stephen Hemminger 已提交
439
/* 4. Try to fixup all. It is made immediately after connection enters
L
Linus Torvalds 已提交
440 441
 *    established state.
 */
442
void tcp_init_buffer_space(struct sock *sk)
L
Linus Torvalds 已提交
443
{
444
	int tcp_app_win = sock_net(sk)->ipv4.sysctl_tcp_app_win;
L
Linus Torvalds 已提交
445 446 447 448 449 450
	struct tcp_sock *tp = tcp_sk(sk);
	int maxwin;

	if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK))
		tcp_fixup_rcvbuf(sk);
	if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK))
E
Eric Dumazet 已提交
451
		tcp_sndbuf_expand(sk);
L
Linus Torvalds 已提交
452 453

	tp->rcvq_space.space = tp->rcv_wnd;
454
	tcp_mstamp_refresh(tp);
455
	tp->rcvq_space.time = tp->tcp_mstamp;
E
Eric Dumazet 已提交
456
	tp->rcvq_space.seq = tp->copied_seq;
L
Linus Torvalds 已提交
457 458 459 460 461 462

	maxwin = tcp_full_space(sk);

	if (tp->window_clamp >= maxwin) {
		tp->window_clamp = maxwin;

463
		if (tcp_app_win && maxwin > 4 * tp->advmss)
L
Linus Torvalds 已提交
464
			tp->window_clamp = max(maxwin -
465
					       (maxwin >> tcp_app_win),
L
Linus Torvalds 已提交
466 467 468 469
					       4 * tp->advmss);
	}

	/* Force reservation of one segment. */
470
	if (tcp_app_win &&
L
Linus Torvalds 已提交
471 472 473 474 475
	    tp->window_clamp > 2 * tp->advmss &&
	    tp->window_clamp + tp->advmss > maxwin)
		tp->window_clamp = max(2 * tp->advmss, maxwin - tp->advmss);

	tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp);
476
	tp->snd_cwnd_stamp = tcp_jiffies32;
L
Linus Torvalds 已提交
477 478 479
}

/* 5. Recalculate window clamp after socket hit its memory bounds. */
480
static void tcp_clamp_window(struct sock *sk)
L
Linus Torvalds 已提交
481
{
482
	struct tcp_sock *tp = tcp_sk(sk);
483
	struct inet_connection_sock *icsk = inet_csk(sk);
484
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
485

486
	icsk->icsk_ack.quick = 0;
L
Linus Torvalds 已提交
487

488
	if (sk->sk_rcvbuf < net->ipv4.sysctl_tcp_rmem[2] &&
489
	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) &&
490
	    !tcp_under_memory_pressure(sk) &&
491
	    sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)) {
492
		sk->sk_rcvbuf = min(atomic_read(&sk->sk_rmem_alloc),
493
				    net->ipv4.sysctl_tcp_rmem[2]);
L
Linus Torvalds 已提交
494
	}
495
	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
496
		tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss);
L
Linus Torvalds 已提交
497 498
}

S
Stephen Hemminger 已提交
499 500 501 502 503 504 505 506 507
/* Initialize RCV_MSS value.
 * RCV_MSS is an our guess about MSS used by the peer.
 * We haven't any direct information about the MSS.
 * It's better to underestimate the RCV_MSS rather than overestimate.
 * Overestimations make us ACKing less frequently than needed.
 * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss().
 */
void tcp_initialize_rcv_mss(struct sock *sk)
{
508
	const struct tcp_sock *tp = tcp_sk(sk);
S
Stephen Hemminger 已提交
509 510
	unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache);

511
	hint = min(hint, tp->rcv_wnd / 2);
512
	hint = min(hint, TCP_MSS_DEFAULT);
S
Stephen Hemminger 已提交
513 514 515 516
	hint = max(hint, TCP_MIN_MSS);

	inet_csk(sk)->icsk_ack.rcv_mss = hint;
}
E
Eric Dumazet 已提交
517
EXPORT_SYMBOL(tcp_initialize_rcv_mss);
S
Stephen Hemminger 已提交
518

L
Linus Torvalds 已提交
519 520 521 522
/* Receiver "autotuning" code.
 *
 * The algorithm for RTT estimation w/o timestamps is based on
 * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL.
523
 * <http://public.lanl.gov/radiant/pubs.html#DRS>
L
Linus Torvalds 已提交
524 525
 *
 * More detail on this code can be found at
526
 * <http://staff.psc.edu/jheffner/>,
L
Linus Torvalds 已提交
527 528 529 530 531
 * though this reference is out of date.  A new paper
 * is pending.
 */
static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)
{
532
	u32 new_sample = tp->rcv_rtt_est.rtt_us;
L
Linus Torvalds 已提交
533 534 535 536 537 538 539 540 541
	long m = sample;

	if (new_sample != 0) {
		/* If we sample in larger samples in the non-timestamp
		 * case, we could grossly overestimate the RTT especially
		 * with chatty applications or bulk transfer apps which
		 * are stalled on filesystem I/O.
		 *
		 * Also, since we are only going for a minimum in the
S
Stephen Hemminger 已提交
542
		 * non-timestamp case, we do not smooth things out
S
Stephen Hemminger 已提交
543
		 * else with timestamps disabled convergence takes too
L
Linus Torvalds 已提交
544 545 546 547 548
		 * long.
		 */
		if (!win_dep) {
			m -= (new_sample >> 3);
			new_sample += m;
549 550 551 552 553
		} else {
			m <<= 3;
			if (m < new_sample)
				new_sample = m;
		}
L
Linus Torvalds 已提交
554
	} else {
S
Stephen Hemminger 已提交
555
		/* No previous measure. */
L
Linus Torvalds 已提交
556 557 558
		new_sample = m << 3;
	}

559
	tp->rcv_rtt_est.rtt_us = new_sample;
L
Linus Torvalds 已提交
560 561 562 563
}

static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp)
{
564 565
	u32 delta_us;

566
	if (tp->rcv_rtt_est.time == 0)
L
Linus Torvalds 已提交
567 568 569
		goto new_measure;
	if (before(tp->rcv_nxt, tp->rcv_rtt_est.seq))
		return;
570
	delta_us = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcv_rtt_est.time);
571 572
	if (!delta_us)
		delta_us = 1;
573
	tcp_rcv_rtt_update(tp, delta_us, 1);
L
Linus Torvalds 已提交
574 575 576

new_measure:
	tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd;
577
	tp->rcv_rtt_est.time = tp->tcp_mstamp;
L
Linus Torvalds 已提交
578 579
}

580 581
static inline void tcp_rcv_rtt_measure_ts(struct sock *sk,
					  const struct sk_buff *skb)
L
Linus Torvalds 已提交
582
{
583
	struct tcp_sock *tp = tcp_sk(sk);
584

L
Linus Torvalds 已提交
585 586
	if (tp->rx_opt.rcv_tsecr &&
	    (TCP_SKB_CB(skb)->end_seq -
587 588
	     TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss)) {
		u32 delta = tcp_time_stamp(tp) - tp->rx_opt.rcv_tsecr;
589
		u32 delta_us;
590

591 592 593
		if (!delta)
			delta = 1;
		delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
594 595
		tcp_rcv_rtt_update(tp, delta_us, 0);
	}
L
Linus Torvalds 已提交
596 597 598 599 600 601 602 603 604
}

/*
 * This function should be called every time data is copied to user space.
 * It calculates the appropriate TCP receive buffer space.
 */
void tcp_rcv_space_adjust(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
605
	u32 copied;
L
Linus Torvalds 已提交
606
	int time;
607

608 609
	trace_tcp_rcv_space_adjust(sk);

610
	tcp_mstamp_refresh(tp);
611
	time = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcvq_space.time);
612
	if (time < (tp->rcv_rtt_est.rtt_us >> 3) || tp->rcv_rtt_est.rtt_us == 0)
L
Linus Torvalds 已提交
613
		return;
614

E
Eric Dumazet 已提交
615 616 617 618 619 620 621 622 623 624 625 626 627 628
	/* Number of bytes copied to user in last RTT */
	copied = tp->copied_seq - tp->rcvq_space.seq;
	if (copied <= tp->rcvq_space.space)
		goto new_measure;

	/* A bit of theory :
	 * copied = bytes received in previous RTT, our base window
	 * To cope with packet losses, we need a 2x factor
	 * To cope with slow start, and sender growing its cwin by 100 %
	 * every RTT, we need a 4x factor, because the ACK we are sending
	 * now is for the next RTT, not the current one :
	 * <prev RTT . ><current RTT .. ><next RTT .... >
	 */

629
	if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf &&
E
Eric Dumazet 已提交
630
	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
631
		int rcvmem, rcvbuf;
E
Eric Dumazet 已提交
632
		u64 rcvwin, grow;
L
Linus Torvalds 已提交
633

E
Eric Dumazet 已提交
634 635 636
		/* minimal window to cope with packet losses, assuming
		 * steady state. Add some cushion because of small variations.
		 */
637
		rcvwin = ((u64)copied << 1) + 16 * tp->advmss;
L
Linus Torvalds 已提交
638

E
Eric Dumazet 已提交
639 640 641 642
		/* Accommodate for sender rate increase (eg. slow start) */
		grow = rcvwin * (copied - tp->rcvq_space.space);
		do_div(grow, tp->rcvq_space.space);
		rcvwin += (grow << 1);
L
Linus Torvalds 已提交
643

E
Eric Dumazet 已提交
644
		rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER);
645
		while (tcp_win_from_space(sk, rcvmem) < tp->advmss)
E
Eric Dumazet 已提交
646
			rcvmem += 128;
L
Linus Torvalds 已提交
647

648 649 650
		do_div(rcvwin, tp->advmss);
		rcvbuf = min_t(u64, rcvwin * rcvmem,
			       sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
E
Eric Dumazet 已提交
651 652
		if (rcvbuf > sk->sk_rcvbuf) {
			sk->sk_rcvbuf = rcvbuf;
L
Linus Torvalds 已提交
653

E
Eric Dumazet 已提交
654
			/* Make the window clamp follow along.  */
655
			tp->window_clamp = tcp_win_from_space(sk, rcvbuf);
L
Linus Torvalds 已提交
656 657
		}
	}
E
Eric Dumazet 已提交
658
	tp->rcvq_space.space = copied;
659

L
Linus Torvalds 已提交
660 661
new_measure:
	tp->rcvq_space.seq = tp->copied_seq;
662
	tp->rcvq_space.time = tp->tcp_mstamp;
L
Linus Torvalds 已提交
663 664 665 666 667 668 669 670 671 672 673 674
}

/* There is something which you must keep in mind when you analyze the
 * behavior of the tp->ato delayed ack timeout interval.  When a
 * connection starts up, we want to ack as quickly as possible.  The
 * problem is that "good" TCP's do slow start at the beginning of data
 * transmission.  The means that until we send the first few ACK's the
 * sender will sit on his end and only queue most of his data, because
 * he can only send snd_cwnd unacked packets at any given time.  For
 * each ACK we send, he increments snd_cwnd and transmits more of his
 * queue.  -DaveM
 */
675
static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
676
{
677
	struct tcp_sock *tp = tcp_sk(sk);
678
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
679 680
	u32 now;

681
	inet_csk_schedule_ack(sk);
L
Linus Torvalds 已提交
682

683
	tcp_measure_rcv_mss(sk, skb);
L
Linus Torvalds 已提交
684 685

	tcp_rcv_rtt_measure(tp);
686

687
	now = tcp_jiffies32;
L
Linus Torvalds 已提交
688

689
	if (!icsk->icsk_ack.ato) {
L
Linus Torvalds 已提交
690 691 692
		/* The _first_ data packet received, initialize
		 * delayed ACK engine.
		 */
693
		tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
694
		icsk->icsk_ack.ato = TCP_ATO_MIN;
L
Linus Torvalds 已提交
695
	} else {
696
		int m = now - icsk->icsk_ack.lrcvtime;
L
Linus Torvalds 已提交
697

698
		if (m <= TCP_ATO_MIN / 2) {
L
Linus Torvalds 已提交
699
			/* The fastest case is the first. */
700 701 702 703 704 705
			icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2;
		} else if (m < icsk->icsk_ack.ato) {
			icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m;
			if (icsk->icsk_ack.ato > icsk->icsk_rto)
				icsk->icsk_ack.ato = icsk->icsk_rto;
		} else if (m > icsk->icsk_rto) {
S
Stephen Hemminger 已提交
706
			/* Too long gap. Apparently sender failed to
L
Linus Torvalds 已提交
707 708
			 * restart window, so that we send ACKs quickly.
			 */
709
			tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
710
			sk_mem_reclaim(sk);
L
Linus Torvalds 已提交
711 712
		}
	}
713
	icsk->icsk_ack.lrcvtime = now;
L
Linus Torvalds 已提交
714

715
	tcp_ecn_check_ce(sk, skb);
L
Linus Torvalds 已提交
716 717

	if (skb->len >= 128)
718
		tcp_grow_window(sk, skb);
L
Linus Torvalds 已提交
719 720 721 722 723 724 725 726 727 728 729
}

/* Called to compute a smoothed rtt estimate. The data fed to this
 * routine either comes from timestamps, or from segments that were
 * known _not_ to have been retransmitted [see Karn/Partridge
 * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
 * piece by Van Jacobson.
 * NOTE: the next three routines used to be one big routine.
 * To save cycles in the RFC 1323 implementation it was better to break
 * it up into three procedures. -- erics
 */
730
static void tcp_rtt_estimator(struct sock *sk, long mrtt_us)
L
Linus Torvalds 已提交
731
{
732
	struct tcp_sock *tp = tcp_sk(sk);
733 734
	long m = mrtt_us; /* RTT */
	u32 srtt = tp->srtt_us;
L
Linus Torvalds 已提交
735 736 737 738

	/*	The following amusing code comes from Jacobson's
	 *	article in SIGCOMM '88.  Note that rtt and mdev
	 *	are scaled versions of rtt and mean deviation.
739
	 *	This is designed to be as fast as possible
L
Linus Torvalds 已提交
740 741 742 743 744 745 746
	 *	m stands for "measurement".
	 *
	 *	On a 1990 paper the rto value is changed to:
	 *	RTO = rtt + 4 * mdev
	 *
	 * Funny. This algorithm seems to be very broken.
	 * These formulae increase RTO, when it should be decreased, increase
S
Stephen Hemminger 已提交
747
	 * too slowly, when it should be increased quickly, decrease too quickly
L
Linus Torvalds 已提交
748 749 750 751
	 * etc. I guess in BSD RTO takes ONE value, so that it is absolutely
	 * does not matter how to _calculate_ it. Seems, it was trap
	 * that VJ failed to avoid. 8)
	 */
752 753 754
	if (srtt != 0) {
		m -= (srtt >> 3);	/* m is now error in rtt est */
		srtt += m;		/* rtt = 7/8 rtt + 1/8 new */
L
Linus Torvalds 已提交
755 756
		if (m < 0) {
			m = -m;		/* m is now abs(error) */
757
			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
L
Linus Torvalds 已提交
758 759 760 761 762 763 764 765 766 767 768
			/* This is similar to one of Eifel findings.
			 * Eifel blocks mdev updates when rtt decreases.
			 * This solution is a bit different: we use finer gain
			 * for mdev in this case (alpha*beta).
			 * Like Eifel it also prevents growth of rto,
			 * but also it limits too fast rto decreases,
			 * happening in pure Eifel.
			 */
			if (m > 0)
				m >>= 3;
		} else {
769
			m -= (tp->mdev_us >> 2);   /* similar update on mdev */
L
Linus Torvalds 已提交
770
		}
771 772 773 774 775
		tp->mdev_us += m;		/* mdev = 3/4 mdev + 1/4 new */
		if (tp->mdev_us > tp->mdev_max_us) {
			tp->mdev_max_us = tp->mdev_us;
			if (tp->mdev_max_us > tp->rttvar_us)
				tp->rttvar_us = tp->mdev_max_us;
L
Linus Torvalds 已提交
776 777
		}
		if (after(tp->snd_una, tp->rtt_seq)) {
778 779
			if (tp->mdev_max_us < tp->rttvar_us)
				tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2;
L
Linus Torvalds 已提交
780
			tp->rtt_seq = tp->snd_nxt;
781
			tp->mdev_max_us = tcp_rto_min_us(sk);
L
Linus Torvalds 已提交
782 783 784
		}
	} else {
		/* no previous measure. */
785
		srtt = m << 3;		/* take the measured time to be rtt */
786 787 788
		tp->mdev_us = m << 1;	/* make sure rto = 3*rtt */
		tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk));
		tp->mdev_max_us = tp->rttvar_us;
L
Linus Torvalds 已提交
789 790
		tp->rtt_seq = tp->snd_nxt;
	}
791
	tp->srtt_us = max(1U, srtt);
L
Linus Torvalds 已提交
792 793
}

E
Eric Dumazet 已提交
794 795 796 797 798 799
static void tcp_update_pacing_rate(struct sock *sk)
{
	const struct tcp_sock *tp = tcp_sk(sk);
	u64 rate;

	/* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
800 801 802 803 804 805 806 807 808 809 810
	rate = (u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3);

	/* current rate is (cwnd * mss) / srtt
	 * In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
	 * In Congestion Avoidance phase, set it to 120 % the current rate.
	 *
	 * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
	 *	 If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
	 *	 end of slow start and should slow down.
	 */
	if (tp->snd_cwnd < tp->snd_ssthresh / 2)
811
		rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio;
812
	else
813
		rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ca_ratio;
E
Eric Dumazet 已提交
814 815 816

	rate *= max(tp->snd_cwnd, tp->packets_out);

817 818
	if (likely(tp->srtt_us))
		do_div(rate, tp->srtt_us);
E
Eric Dumazet 已提交
819

820
	/* WRITE_ONCE() is needed because sch_fq fetches sk_pacing_rate
821 822 823
	 * without any lock. We want to make sure compiler wont store
	 * intermediate values in this location.
	 */
824 825
	WRITE_ONCE(sk->sk_pacing_rate, min_t(u64, rate,
					     sk->sk_max_pacing_rate));
E
Eric Dumazet 已提交
826 827
}

L
Linus Torvalds 已提交
828 829 830
/* Calculate rto without backoff.  This is the second half of Van Jacobson's
 * routine referred to above.
 */
831
static void tcp_set_rto(struct sock *sk)
L
Linus Torvalds 已提交
832
{
833
	const struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
834 835 836 837 838 839 840 841
	/* Old crap is replaced with new one. 8)
	 *
	 * More seriously:
	 * 1. If rtt variance happened to be less 50msec, it is hallucination.
	 *    It cannot be less due to utterly erratic ACK generation made
	 *    at least by solaris and freebsd. "Erratic ACKs" has _nothing_
	 *    to do with delayed acks, because at cwnd>2 true delack timeout
	 *    is invisible. Actually, Linux-2.4 also generates erratic
S
Stephen Hemminger 已提交
842
	 *    ACKs in some circumstances.
L
Linus Torvalds 已提交
843
	 */
844
	inet_csk(sk)->icsk_rto = __tcp_set_rto(tp);
L
Linus Torvalds 已提交
845 846 847 848

	/* 2. Fixups made earlier cannot be right.
	 *    If we do not estimate RTO correctly without them,
	 *    all the algo is pure shit and should be replaced
S
Stephen Hemminger 已提交
849
	 *    with correct one. It is exactly, which we pretend to do.
L
Linus Torvalds 已提交
850 851
	 */

852 853 854
	/* NOTE: clamping at TCP_RTO_MIN is not required, current algo
	 * guarantees that rto is higher.
	 */
855
	tcp_bound_rto(sk);
L
Linus Torvalds 已提交
856 857
}

858
__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst)
L
Linus Torvalds 已提交
859 860 861
{
	__u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0);

862
	if (!cwnd)
863
		cwnd = TCP_INIT_CWND;
L
Linus Torvalds 已提交
864 865 866
	return min_t(__u32, cwnd, tp->snd_cwnd_clamp);
}

867
/* Take a notice that peer is sending D-SACKs */
868 869
static void tcp_dsack_seen(struct tcp_sock *tp)
{
870
	tp->rx_opt.sack_ok |= TCP_DSACK_SEEN;
871
	tp->rack.dsack_seen = 1;
872 873
}

874 875 876 877 878 879
/* It's reordering when higher sequence was delivered (i.e. sacked) before
 * some lower never-retransmitted sequence ("low_seq"). The maximum reordering
 * distance is approximated in full-mss packet distance ("reordering").
 */
static void tcp_check_sack_reordering(struct sock *sk, const u32 low_seq,
				      const int ts)
L
Linus Torvalds 已提交
880
{
881
	struct tcp_sock *tp = tcp_sk(sk);
882 883
	const u32 mss = tp->mss_cache;
	u32 fack, metric;
884

885 886
	fack = tcp_highest_sack_seq(tp);
	if (!before(low_seq, fack))
887 888
		return;

889 890
	metric = fack - low_seq;
	if ((metric > tp->reordering * mss) && mss) {
L
Linus Torvalds 已提交
891
#if FASTRETRANS_DEBUG > 1
892 893 894
		pr_debug("Disorder%d %d %u f%u s%u rr%d\n",
			 tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state,
			 tp->reordering,
895
			 0,
896 897
			 tp->sacked_out,
			 tp->undo_marker ? tp->undo_retrans : 0);
L
Linus Torvalds 已提交
898
#endif
899 900
		tp->reordering = min_t(u32, (metric + mss - 1) / mss,
				       sock_net(sk)->ipv4.sysctl_tcp_max_reordering);
L
Linus Torvalds 已提交
901
	}
Y
Yuchung Cheng 已提交
902

Y
Yuchung Cheng 已提交
903
	tp->rack.reord = 1;
904
	/* This exciting event is worth to be remembered. 8) */
905 906
	NET_INC_STATS(sock_net(sk),
		      ts ? LINUX_MIB_TCPTSREORDER : LINUX_MIB_TCPSACKREORDER);
L
Linus Torvalds 已提交
907 908
}

909
/* This must be called before lost_out is incremented */
910 911
static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)
{
912
	if (!tp->retransmit_skb_hint ||
913 914
	    before(TCP_SKB_CB(skb)->seq,
		   TCP_SKB_CB(tp->retransmit_skb_hint)->seq))
915
		tp->retransmit_skb_hint = skb;
916 917
}

918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933
/* Sum the number of packets on the wire we have marked as lost.
 * There are two cases we care about here:
 * a) Packet hasn't been marked lost (nor retransmitted),
 *    and this is the first loss.
 * b) Packet has been marked both lost and retransmitted,
 *    and this means we think it was lost again.
 */
static void tcp_sum_lost(struct tcp_sock *tp, struct sk_buff *skb)
{
	__u8 sacked = TCP_SKB_CB(skb)->sacked;

	if (!(sacked & TCPCB_LOST) ||
	    ((sacked & TCPCB_LOST) && (sacked & TCPCB_SACKED_RETRANS)))
		tp->lost += tcp_skb_pcount(skb);
}

934 935 936 937 938 939
static void tcp_skb_mark_lost(struct tcp_sock *tp, struct sk_buff *skb)
{
	if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) {
		tcp_verify_retransmit_hint(tp, skb);

		tp->lost_out += tcp_skb_pcount(skb);
940
		tcp_sum_lost(tp, skb);
941 942 943 944
		TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
	}
}

Y
Yuchung Cheng 已提交
945
void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb)
946 947 948
{
	tcp_verify_retransmit_hint(tp, skb);

949
	tcp_sum_lost(tp, skb);
950 951 952 953 954 955
	if (!(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_ACKED))) {
		tp->lost_out += tcp_skb_pcount(skb);
		TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
	}
}

L
Linus Torvalds 已提交
956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
/* This procedure tags the retransmission queue when SACKs arrive.
 *
 * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L).
 * Packets in queue with these bits set are counted in variables
 * sacked_out, retrans_out and lost_out, correspondingly.
 *
 * Valid combinations are:
 * Tag  InFlight	Description
 * 0	1		- orig segment is in flight.
 * S	0		- nothing flies, orig reached receiver.
 * L	0		- nothing flies, orig lost by net.
 * R	2		- both orig and retransmit are in flight.
 * L|R	1		- orig is lost, retransmit is in flight.
 * S|R  1		- orig reached receiver, retrans is still in flight.
 * (L|S|R is logically valid, it could occur when L|R is sacked,
 *  but it is equivalent to plain S and code short-curcuits it to S.
 *  L|S is logically invalid, it would mean -1 packet in flight 8))
 *
 * These 6 states form finite state machine, controlled by the following events:
 * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue())
 * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue())
977
 * 3. Loss detection event of two flavors:
L
Linus Torvalds 已提交
978 979
 *	A. Scoreboard estimator decided the packet is lost.
 *	   A'. Reno "three dupacks" marks head of queue lost.
980
 *	B. SACK arrives sacking SND.NXT at the moment, when the
L
Linus Torvalds 已提交
981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999
 *	   segment was retransmitted.
 * 4. D-SACK added new rule: D-SACK changes any tag to S.
 *
 * It is pleasant to note, that state diagram turns out to be commutative,
 * so that we are allowed not to be bothered by order of our actions,
 * when multiple events arrive simultaneously. (see the function below).
 *
 * Reordering detection.
 * --------------------
 * Reordering metric is maximal distance, which a packet can be displaced
 * in packet stream. With SACKs we can estimate it:
 *
 * 1. SACK fills old hole and the corresponding segment was not
 *    ever retransmitted -> reordering. Alas, we cannot use it
 *    when segment was retransmitted.
 * 2. The last flaw is solved with D-SACK. D-SACK arrives
 *    for retransmitted and already SACKed segment -> reordering..
 * Both of these heuristics are not used in Loss state, when we cannot
 * account for retransmits accurately.
I
Ilpo Järvinen 已提交
1000 1001 1002 1003 1004 1005 1006
 *
 * SACK block validation.
 * ----------------------
 *
 * SACK block range validation checks that the received SACK block fits to
 * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT.
 * Note that SND.UNA is not included to the range though being valid because
1007 1008 1009 1010 1011 1012 1013 1014 1015
 * it means that the receiver is rather inconsistent with itself reporting
 * SACK reneging when it should advance SND.UNA. Such SACK block this is
 * perfectly valid, however, in light of RFC2018 which explicitly states
 * that "SACK block MUST reflect the newest segment.  Even if the newest
 * segment is going to be discarded ...", not that it looks very clever
 * in case of head skb. Due to potentional receiver driven attacks, we
 * choose to avoid immediate execution of a walk in write queue due to
 * reneging and defer head skb's loss recovery to standard loss recovery
 * procedure that will eventually trigger (nothing forbids us doing this).
I
Ilpo Järvinen 已提交
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
 *
 * Implements also blockage to start_seq wrap-around. Problem lies in the
 * fact that though start_seq (s) is before end_seq (i.e., not reversed),
 * there's no guarantee that it will be before snd_nxt (n). The problem
 * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt
 * wrap (s_w):
 *
 *         <- outs wnd ->                          <- wrapzone ->
 *         u     e      n                         u_w   e_w  s n_w
 *         |     |      |                          |     |   |  |
 * |<------------+------+----- TCP seqno space --------------+---------->|
 * ...-- <2^31 ->|                                           |<--------...
 * ...---- >2^31 ------>|                                    |<--------...
 *
 * Current code wouldn't be vulnerable but it's better still to discard such
 * crazy SACK blocks. Doing this check for start_seq alone closes somewhat
 * similar case (end_seq after snd_nxt wrap) as earlier reversed check in
 * snd_nxt wrap -> snd_una region will then become "well defined", i.e.,
 * equal to the ideal case (infinite seqno space without wrap caused issues).
 *
 * With D-SACK the lower bound is extended to cover sequence space below
 * SND.UNA down to undo_marker, which is the last point of interest. Yet
1038
 * again, D-SACK block must not to go across snd_una (for the same reason as
I
Ilpo Järvinen 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047
 * for the normal SACK blocks, explained above). But there all simplicity
 * ends, TCP might receive valid D-SACKs below that. As long as they reside
 * fully below undo_marker they do not affect behavior in anyway and can
 * therefore be safely ignored. In rare cases (which are more or less
 * theoretical ones), the D-SACK will nicely cross that boundary due to skb
 * fragmentation and packet reordering past skb's retransmission. To consider
 * them correctly, the acceptable range must be extended even more though
 * the exact amount is rather hard to quantify. However, tp->max_window can
 * be used as an exaggerated estimate.
L
Linus Torvalds 已提交
1048
 */
E
Eric Dumazet 已提交
1049 1050
static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack,
				   u32 start_seq, u32 end_seq)
I
Ilpo Järvinen 已提交
1051 1052 1053
{
	/* Too far in future, or reversed (interpretation is ambiguous) */
	if (after(end_seq, tp->snd_nxt) || !before(start_seq, end_seq))
E
Eric Dumazet 已提交
1054
		return false;
I
Ilpo Järvinen 已提交
1055 1056 1057

	/* Nasty start_seq wrap-around check (see comments above) */
	if (!before(start_seq, tp->snd_nxt))
E
Eric Dumazet 已提交
1058
		return false;
I
Ilpo Järvinen 已提交
1059

1060
	/* In outstanding window? ...This is valid exit for D-SACKs too.
I
Ilpo Järvinen 已提交
1061 1062 1063
	 * start_seq == snd_una is non-sensical (see comments above)
	 */
	if (after(start_seq, tp->snd_una))
E
Eric Dumazet 已提交
1064
		return true;
I
Ilpo Järvinen 已提交
1065 1066

	if (!is_dsack || !tp->undo_marker)
E
Eric Dumazet 已提交
1067
		return false;
I
Ilpo Järvinen 已提交
1068 1069

	/* ...Then it's D-SACK, and must reside below snd_una completely */
Z
Zheng Yan 已提交
1070
	if (after(end_seq, tp->snd_una))
E
Eric Dumazet 已提交
1071
		return false;
I
Ilpo Järvinen 已提交
1072 1073

	if (!before(start_seq, tp->undo_marker))
E
Eric Dumazet 已提交
1074
		return true;
I
Ilpo Järvinen 已提交
1075 1076 1077

	/* Too old */
	if (!after(end_seq, tp->undo_marker))
E
Eric Dumazet 已提交
1078
		return false;
I
Ilpo Järvinen 已提交
1079 1080 1081 1082 1083 1084 1085

	/* Undo_marker boundary crossing (overestimates a lot). Known already:
	 *   start_seq < undo_marker and end_seq >= undo_marker.
	 */
	return !before(start_seq, end_seq - tp->max_window);
}

E
Eric Dumazet 已提交
1086 1087 1088
static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb,
			    struct tcp_sack_block_wire *sp, int num_sacks,
			    u32 prior_snd_una)
1089
{
1090
	struct tcp_sock *tp = tcp_sk(sk);
1091 1092
	u32 start_seq_0 = get_unaligned_be32(&sp[0].start_seq);
	u32 end_seq_0 = get_unaligned_be32(&sp[0].end_seq);
E
Eric Dumazet 已提交
1093
	bool dup_sack = false;
1094 1095

	if (before(start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) {
E
Eric Dumazet 已提交
1096
		dup_sack = true;
1097
		tcp_dsack_seen(tp);
1098
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECV);
1099
	} else if (num_sacks > 1) {
1100 1101
		u32 end_seq_1 = get_unaligned_be32(&sp[1].end_seq);
		u32 start_seq_1 = get_unaligned_be32(&sp[1].start_seq);
1102 1103 1104

		if (!after(end_seq_0, end_seq_1) &&
		    !before(start_seq_0, start_seq_1)) {
E
Eric Dumazet 已提交
1105
			dup_sack = true;
1106
			tcp_dsack_seen(tp);
1107
			NET_INC_STATS(sock_net(sk),
1108
					LINUX_MIB_TCPDSACKOFORECV);
1109 1110 1111 1112
		}
	}

	/* D-SACK for already forgotten data... Do dumb counting. */
Y
Yuchung Cheng 已提交
1113
	if (dup_sack && tp->undo_marker && tp->undo_retrans > 0 &&
1114 1115 1116 1117 1118 1119 1120
	    !after(end_seq_0, prior_snd_una) &&
	    after(end_seq_0, tp->undo_marker))
		tp->undo_retrans--;

	return dup_sack;
}

1121
struct tcp_sacktag_state {
1122
	u32	reord;
1123 1124 1125 1126
	/* Timestamps for earliest and latest never-retransmitted segment
	 * that was SACKed. RTO needs the earliest RTT to stay conservative,
	 * but congestion control should still get an accurate delay signal.
	 */
1127 1128
	u64	first_sackt;
	u64	last_sackt;
1129
	struct rate_sample *rate;
1130
	int	flag;
1131
	unsigned int mss_now;
1132 1133
};

1134 1135 1136 1137 1138
/* Check if skb is fully within the SACK block. In presence of GSO skbs,
 * the incoming SACK may not exactly match but we can find smaller MSS
 * aligned portion of it that matches. Therefore we might need to fragment
 * which may fail and creates some hassle (caller must handle error case
 * returns).
1139 1140
 *
 * FIXME: this could be merged to shift decision code
1141
 */
1142
static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb,
E
Eric Dumazet 已提交
1143
				  u32 start_seq, u32 end_seq)
1144
{
E
Eric Dumazet 已提交
1145 1146
	int err;
	bool in_sack;
1147
	unsigned int pkt_len;
1148
	unsigned int mss;
1149 1150 1151 1152 1153 1154

	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);

	if (tcp_skb_pcount(skb) > 1 && !in_sack &&
	    after(TCP_SKB_CB(skb)->end_seq, start_seq)) {
1155
		mss = tcp_skb_mss(skb);
1156 1157
		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);

1158
		if (!in_sack) {
1159
			pkt_len = start_seq - TCP_SKB_CB(skb)->seq;
1160 1161 1162
			if (pkt_len < mss)
				pkt_len = mss;
		} else {
1163
			pkt_len = end_seq - TCP_SKB_CB(skb)->seq;
1164 1165 1166 1167 1168 1169 1170 1171 1172
			if (pkt_len < mss)
				return -EINVAL;
		}

		/* Round if necessary so that SACKs cover only full MSSes
		 * and/or the remaining small portion (if present)
		 */
		if (pkt_len > mss) {
			unsigned int new_len = (pkt_len / mss) * mss;
1173
			if (!in_sack && new_len < pkt_len)
1174 1175 1176
				new_len += mss;
			pkt_len = new_len;
		}
1177 1178 1179 1180

		if (pkt_len >= skb->len && !in_sack)
			return 0;

1181 1182
		err = tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
				   pkt_len, mss, GFP_ATOMIC);
1183 1184 1185 1186 1187 1188 1189
		if (err < 0)
			return err;
	}

	return in_sack;
}

1190 1191 1192 1193
/* Mark the given newly-SACKed range as such, adjusting counters and hints. */
static u8 tcp_sacktag_one(struct sock *sk,
			  struct tcp_sacktag_state *state, u8 sacked,
			  u32 start_seq, u32 end_seq,
1194
			  int dup_sack, int pcount,
1195
			  u64 xmit_time)
1196
{
1197
	struct tcp_sock *tp = tcp_sk(sk);
1198 1199 1200

	/* Account D-SACK for retransmitted packet. */
	if (dup_sack && (sacked & TCPCB_RETRANS)) {
Y
Yuchung Cheng 已提交
1201
		if (tp->undo_marker && tp->undo_retrans > 0 &&
1202
		    after(end_seq, tp->undo_marker))
1203
			tp->undo_retrans--;
1204 1205 1206
		if ((sacked & TCPCB_SACKED_ACKED) &&
		    before(start_seq, state->reord))
				state->reord = start_seq;
1207 1208 1209
	}

	/* Nothing to do; acked frame is about to be dropped (was ACKed). */
1210
	if (!after(end_seq, tp->snd_una))
1211
		return sacked;
1212 1213

	if (!(sacked & TCPCB_SACKED_ACKED)) {
1214
		tcp_rack_advance(tp, sacked, end_seq, xmit_time);
1215

1216 1217 1218 1219 1220 1221
		if (sacked & TCPCB_SACKED_RETRANS) {
			/* If the segment is not tagged as lost,
			 * we do not clear RETRANS, believing
			 * that retransmission is still in flight.
			 */
			if (sacked & TCPCB_LOST) {
1222
				sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS);
1223 1224
				tp->lost_out -= pcount;
				tp->retrans_out -= pcount;
1225 1226 1227 1228 1229 1230
			}
		} else {
			if (!(sacked & TCPCB_RETRANS)) {
				/* New sack for not retransmitted frame,
				 * which was in hole. It is reordering.
				 */
1231
				if (before(start_seq,
1232 1233 1234 1235
					   tcp_highest_sack_seq(tp)) &&
				    before(start_seq, state->reord))
					state->reord = start_seq;

Y
Yuchung Cheng 已提交
1236 1237
				if (!after(end_seq, tp->high_seq))
					state->flag |= FLAG_ORIG_SACK_ACKED;
1238 1239 1240
				if (state->first_sackt == 0)
					state->first_sackt = xmit_time;
				state->last_sackt = xmit_time;
1241 1242 1243
			}

			if (sacked & TCPCB_LOST) {
1244
				sacked &= ~TCPCB_LOST;
1245
				tp->lost_out -= pcount;
1246 1247 1248
			}
		}

1249 1250
		sacked |= TCPCB_SACKED_ACKED;
		state->flag |= FLAG_DATA_SACKED;
1251
		tp->sacked_out += pcount;
Y
Yuchung Cheng 已提交
1252
		tp->delivered += pcount;  /* Out-of-order packets delivered */
1253 1254

		/* Lost marker hint past SACKed? Tweak RFC3517 cnt */
Y
Yuchung Cheng 已提交
1255
		if (tp->lost_skb_hint &&
1256
		    before(start_seq, TCP_SKB_CB(tp->lost_skb_hint)->seq))
1257
			tp->lost_cnt_hint += pcount;
1258 1259 1260 1261 1262 1263
	}

	/* D-SACK. We can detect redundant retransmission in S|R and plain R
	 * frames and clear it. undo_retrans is decreased above, L|R frames
	 * are accounted above as well.
	 */
1264 1265
	if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) {
		sacked &= ~TCPCB_SACKED_RETRANS;
1266
		tp->retrans_out -= pcount;
1267 1268
	}

1269
	return sacked;
1270 1271
}

1272 1273 1274
/* Shift newly-SACKed bytes from this skb to the immediately previous
 * already-SACKed sk_buff. Mark the newly-SACKed bytes as such.
 */
1275 1276
static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev,
			    struct sk_buff *skb,
E
Eric Dumazet 已提交
1277 1278 1279
			    struct tcp_sacktag_state *state,
			    unsigned int pcount, int shifted, int mss,
			    bool dup_sack)
1280 1281
{
	struct tcp_sock *tp = tcp_sk(sk);
1282 1283
	u32 start_seq = TCP_SKB_CB(skb)->seq;	/* start of newly-SACKed */
	u32 end_seq = start_seq + shifted;	/* end of newly-SACKed */
1284 1285 1286

	BUG_ON(!pcount);

1287 1288 1289 1290 1291 1292 1293
	/* Adjust counters and hints for the newly sacked sequence
	 * range but discard the return value since prev is already
	 * marked. We must tag the range first because the seq
	 * advancement below implicitly advances
	 * tcp_highest_sack_seq() when skb is highest_sack.
	 */
	tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked,
Y
Yuchung Cheng 已提交
1294
			start_seq, end_seq, dup_sack, pcount,
1295
			skb->skb_mstamp);
1296
	tcp_rate_skb_delivered(sk, skb, state->rate);
1297 1298

	if (skb == tp->lost_skb_hint)
1299 1300
		tp->lost_cnt_hint += pcount;

1301 1302 1303
	TCP_SKB_CB(prev)->end_seq += shifted;
	TCP_SKB_CB(skb)->seq += shifted;

1304 1305 1306
	tcp_skb_pcount_add(prev, pcount);
	BUG_ON(tcp_skb_pcount(skb) < pcount);
	tcp_skb_pcount_add(skb, -pcount);
1307 1308 1309 1310 1311 1312

	/* When we're adding to gso_segs == 1, gso_size will be zero,
	 * in theory this shouldn't be necessary but as long as DSACK
	 * code can come after this skb later on it's better to keep
	 * setting gso_size to something.
	 */
1313 1314
	if (!TCP_SKB_CB(prev)->tcp_gso_size)
		TCP_SKB_CB(prev)->tcp_gso_size = mss;
1315 1316

	/* CHECKME: To clear or not to clear? Mimics normal skb currently */
1317
	if (tcp_skb_pcount(skb) <= 1)
1318
		TCP_SKB_CB(skb)->tcp_gso_size = 0;
1319 1320 1321 1322 1323 1324

	/* Difference in this won't matter, both ACKed by the same cumul. ACK */
	TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS);

	if (skb->len > 0) {
		BUG_ON(!tcp_skb_pcount(skb));
1325
		NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTED);
E
Eric Dumazet 已提交
1326
		return false;
1327 1328 1329 1330
	}

	/* Whole SKB was eaten :-) */

1331 1332 1333 1334 1335 1336 1337
	if (skb == tp->retransmit_skb_hint)
		tp->retransmit_skb_hint = prev;
	if (skb == tp->lost_skb_hint) {
		tp->lost_skb_hint = prev;
		tp->lost_cnt_hint -= tcp_skb_pcount(prev);
	}

1338
	TCP_SKB_CB(prev)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1339
	TCP_SKB_CB(prev)->eor = TCP_SKB_CB(skb)->eor;
1340 1341 1342
	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
		TCP_SKB_CB(prev)->end_seq++;

1343 1344 1345
	if (skb == tcp_highest_sack(sk))
		tcp_advance_highest_sack(sk, skb);

1346
	tcp_skb_collapse_tstamp(prev, skb);
1347 1348
	if (unlikely(TCP_SKB_CB(prev)->tx.delivered_mstamp))
		TCP_SKB_CB(prev)->tx.delivered_mstamp = 0;
1349

1350
	tcp_rtx_queue_unlink_and_free(skb, sk);
1351

1352
	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKMERGED);
1353

E
Eric Dumazet 已提交
1354
	return true;
1355 1356 1357 1358 1359
}

/* I wish gso_size would have a bit more sane initialization than
 * something-or-zero which complicates things
 */
1360
static int tcp_skb_seglen(const struct sk_buff *skb)
1361
{
1362
	return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb);
1363 1364 1365
}

/* Shifting pages past head area doesn't work */
1366
static int skb_can_shift(const struct sk_buff *skb)
1367 1368 1369 1370 1371 1372 1373 1374
{
	return !skb_headlen(skb) && skb_is_nonlinear(skb);
}

/* Try collapsing SACK blocks spanning across multiple skbs to a single
 * skb.
 */
static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb,
1375
					  struct tcp_sacktag_state *state,
1376
					  u32 start_seq, u32 end_seq,
E
Eric Dumazet 已提交
1377
					  bool dup_sack)
1378 1379 1380 1381 1382 1383 1384 1385 1386 1387
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *prev;
	int mss;
	int pcount = 0;
	int len;
	int in_sack;

	/* Normally R but no L won't result in plain S */
	if (!dup_sack &&
1388
	    (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS)
1389 1390 1391 1392 1393 1394 1395 1396
		goto fallback;
	if (!skb_can_shift(skb))
		goto fallback;
	/* This frame is about to be dropped (was ACKed). */
	if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
		goto fallback;

	/* Can only happen with delayed DSACK + discard craziness */
1397 1398
	prev = skb_rb_prev(skb);
	if (!prev)
1399 1400 1401 1402 1403
		goto fallback;

	if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED)
		goto fallback;

1404 1405 1406
	if (!tcp_skb_can_collapse_to(prev))
		goto fallback;

1407 1408 1409 1410 1411 1412
	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
		  !before(end_seq, TCP_SKB_CB(skb)->end_seq);

	if (in_sack) {
		len = skb->len;
		pcount = tcp_skb_pcount(skb);
1413
		mss = tcp_skb_seglen(skb);
1414 1415 1416 1417

		/* TODO: Fix DSACKs to not fragment already SACKed and we can
		 * drop this restriction as unnecessary
		 */
1418
		if (mss != tcp_skb_seglen(prev))
1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
			goto fallback;
	} else {
		if (!after(TCP_SKB_CB(skb)->end_seq, start_seq))
			goto noop;
		/* CHECKME: This is non-MSS split case only?, this will
		 * cause skipped skbs due to advancing loop btw, original
		 * has that feature too
		 */
		if (tcp_skb_pcount(skb) <= 1)
			goto noop;

		in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
		if (!in_sack) {
			/* TODO: head merge to next could be attempted here
			 * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)),
			 * though it might not be worth of the additional hassle
			 *
			 * ...we can probably just fallback to what was done
			 * previously. We could try merging non-SACKed ones
			 * as well but it probably isn't going to buy off
			 * because later SACKs might again split them, and
			 * it would make skb timestamp tracking considerably
			 * harder problem.
			 */
			goto fallback;
		}

		len = end_seq - TCP_SKB_CB(skb)->seq;
		BUG_ON(len < 0);
		BUG_ON(len > skb->len);

		/* MSS boundaries should be honoured or else pcount will
		 * severely break even though it makes things bit trickier.
		 * Optimize common case to avoid most of the divides
		 */
		mss = tcp_skb_mss(skb);

		/* TODO: Fix DSACKs to not fragment already SACKed and we can
		 * drop this restriction as unnecessary
		 */
1459
		if (mss != tcp_skb_seglen(prev))
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
			goto fallback;

		if (len == mss) {
			pcount = 1;
		} else if (len < mss) {
			goto noop;
		} else {
			pcount = len / mss;
			len = pcount * mss;
		}
	}

1472 1473 1474 1475
	/* tcp_sacktag_one() won't SACK-tag ranges below snd_una */
	if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una))
		goto fallback;

1476 1477
	if (!skb_shift(prev, skb, len))
		goto fallback;
1478
	if (!tcp_shifted_skb(sk, prev, skb, state, pcount, len, mss, dup_sack))
1479 1480 1481 1482 1483
		goto out;

	/* Hole filled allows collapsing with the next as well, this is very
	 * useful when hole on every nth skb pattern happens
	 */
1484 1485
	skb = skb_rb_next(prev);
	if (!skb)
1486 1487
		goto out;

1488 1489
	if (!skb_can_shift(skb) ||
	    ((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) ||
1490
	    (mss != tcp_skb_seglen(skb)))
1491 1492 1493 1494 1495
		goto out;

	len = skb->len;
	if (skb_shift(prev, skb, len)) {
		pcount += tcp_skb_pcount(skb);
1496 1497
		tcp_shifted_skb(sk, prev, skb, state, tcp_skb_pcount(skb),
				len, mss, 0);
1498 1499 1500 1501 1502 1503 1504 1505 1506
	}

out:
	return prev;

noop:
	return skb;

fallback:
1507
	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK);
1508 1509 1510
	return NULL;
}

1511 1512
static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk,
					struct tcp_sack_block *next_dup,
1513
					struct tcp_sacktag_state *state,
1514
					u32 start_seq, u32 end_seq,
E
Eric Dumazet 已提交
1515
					bool dup_sack_in)
1516
{
1517 1518 1519
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *tmp;

1520
	skb_rbtree_walk_from(skb) {
1521
		int in_sack = 0;
E
Eric Dumazet 已提交
1522
		bool dup_sack = dup_sack_in;
1523 1524 1525 1526 1527

		/* queue is in-order => we can short-circuit the walk early */
		if (!before(TCP_SKB_CB(skb)->seq, end_seq))
			break;

1528
		if (next_dup  &&
1529 1530 1531 1532 1533
		    before(TCP_SKB_CB(skb)->seq, next_dup->end_seq)) {
			in_sack = tcp_match_skb_to_sack(sk, skb,
							next_dup->start_seq,
							next_dup->end_seq);
			if (in_sack > 0)
E
Eric Dumazet 已提交
1534
				dup_sack = true;
1535 1536
		}

1537 1538 1539 1540 1541
		/* skb reference here is a bit tricky to get right, since
		 * shifting can eat and free both this skb and the next,
		 * so not even _safe variant of the loop is enough.
		 */
		if (in_sack <= 0) {
1542 1543
			tmp = tcp_shift_skb_data(sk, skb, state,
						 start_seq, end_seq, dup_sack);
1544
			if (tmp) {
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557
				if (tmp != skb) {
					skb = tmp;
					continue;
				}

				in_sack = 0;
			} else {
				in_sack = tcp_match_skb_to_sack(sk, skb,
								start_seq,
								end_seq);
			}
		}

1558 1559 1560
		if (unlikely(in_sack < 0))
			break;

1561
		if (in_sack) {
1562 1563 1564 1565 1566 1567 1568
			TCP_SKB_CB(skb)->sacked =
				tcp_sacktag_one(sk,
						state,
						TCP_SKB_CB(skb)->sacked,
						TCP_SKB_CB(skb)->seq,
						TCP_SKB_CB(skb)->end_seq,
						dup_sack,
Y
Yuchung Cheng 已提交
1569
						tcp_skb_pcount(skb),
1570
						skb->skb_mstamp);
1571
			tcp_rate_skb_delivered(sk, skb, state->rate);
1572 1573
			if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
				list_del_init(&skb->tcp_tsorted_anchor);
1574

1575 1576 1577 1578
			if (!before(TCP_SKB_CB(skb)->seq,
				    tcp_highest_sack_seq(tp)))
				tcp_advance_highest_sack(sk, skb);
		}
1579 1580 1581 1582
	}
	return skb;
}

1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605
static struct sk_buff *tcp_sacktag_bsearch(struct sock *sk,
					   struct tcp_sacktag_state *state,
					   u32 seq)
{
	struct rb_node *parent, **p = &sk->tcp_rtx_queue.rb_node;
	struct sk_buff *skb;

	while (*p) {
		parent = *p;
		skb = rb_to_skb(parent);
		if (before(seq, TCP_SKB_CB(skb)->seq)) {
			p = &parent->rb_left;
			continue;
		}
		if (!before(seq, TCP_SKB_CB(skb)->end_seq)) {
			p = &parent->rb_right;
			continue;
		}
		return skb;
	}
	return NULL;
}

1606
static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk,
1607 1608
					struct tcp_sacktag_state *state,
					u32 skip_to_seq)
1609
{
1610 1611
	if (skb && after(TCP_SKB_CB(skb)->seq, skip_to_seq))
		return skb;
1612

1613
	return tcp_sacktag_bsearch(sk, state, skip_to_seq);
1614 1615 1616 1617 1618
}

static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb,
						struct sock *sk,
						struct tcp_sack_block *next_dup,
1619 1620
						struct tcp_sacktag_state *state,
						u32 skip_to_seq)
1621
{
1622
	if (!next_dup)
1623 1624 1625
		return skb;

	if (before(next_dup->start_seq, skip_to_seq)) {
1626 1627 1628 1629
		skb = tcp_sacktag_skip(skb, sk, state, next_dup->start_seq);
		skb = tcp_sacktag_walk(skb, sk, NULL, state,
				       next_dup->start_seq, next_dup->end_seq,
				       1);
1630 1631 1632 1633 1634
	}

	return skb;
}

1635
static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache)
1636 1637 1638 1639
{
	return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
}

L
Linus Torvalds 已提交
1640
static int
1641
tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb,
1642
			u32 prior_snd_una, struct tcp_sacktag_state *state)
L
Linus Torvalds 已提交
1643 1644
{
	struct tcp_sock *tp = tcp_sk(sk);
1645 1646
	const unsigned char *ptr = (skb_transport_header(ack_skb) +
				    TCP_SKB_CB(ack_skb)->sacked);
1647
	struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2);
1648
	struct tcp_sack_block sp[TCP_NUM_SACKS];
1649 1650
	struct tcp_sack_block *cache;
	struct sk_buff *skb;
1651
	int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3);
1652
	int used_sacks;
E
Eric Dumazet 已提交
1653
	bool found_dup_sack = false;
1654
	int i, j;
1655
	int first_sack_index;
L
Linus Torvalds 已提交
1656

1657
	state->flag = 0;
1658
	state->reord = tp->snd_nxt;
1659

1660
	if (!tp->sacked_out)
1661
		tcp_highest_sack_reset(sk);
L
Linus Torvalds 已提交
1662

1663
	found_dup_sack = tcp_check_dsack(sk, ack_skb, sp_wire,
1664
					 num_sacks, prior_snd_una);
1665
	if (found_dup_sack) {
1666
		state->flag |= FLAG_DSACKING_ACK;
1667 1668
		tp->delivered++; /* A spurious retransmission is delivered */
	}
1669 1670 1671 1672 1673 1674 1675 1676

	/* Eliminate too old ACKs, but take into
	 * account more or less fresh ones, they can
	 * contain valid SACK info.
	 */
	if (before(TCP_SKB_CB(ack_skb)->ack_seq, prior_snd_una - tp->max_window))
		return 0;

1677 1678 1679
	if (!tp->packets_out)
		goto out;

1680 1681 1682
	used_sacks = 0;
	first_sack_index = 0;
	for (i = 0; i < num_sacks; i++) {
E
Eric Dumazet 已提交
1683
		bool dup_sack = !i && found_dup_sack;
1684

1685 1686
		sp[used_sacks].start_seq = get_unaligned_be32(&sp_wire[i].start_seq);
		sp[used_sacks].end_seq = get_unaligned_be32(&sp_wire[i].end_seq);
1687 1688 1689 1690

		if (!tcp_is_sackblock_valid(tp, dup_sack,
					    sp[used_sacks].start_seq,
					    sp[used_sacks].end_seq)) {
1691 1692
			int mib_idx;

1693 1694
			if (dup_sack) {
				if (!tp->undo_marker)
1695
					mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO;
1696
				else
1697
					mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD;
1698 1699 1700 1701 1702
			} else {
				/* Don't count olds caused by ACK reordering */
				if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) &&
				    !after(sp[used_sacks].end_seq, tp->snd_una))
					continue;
1703
				mib_idx = LINUX_MIB_TCPSACKDISCARD;
1704
			}
1705

1706
			NET_INC_STATS(sock_net(sk), mib_idx);
1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718
			if (i == 0)
				first_sack_index = -1;
			continue;
		}

		/* Ignore very old stuff early */
		if (!after(sp[used_sacks].end_seq, prior_snd_una))
			continue;

		used_sacks++;
	}

1719 1720
	/* order SACK blocks to allow in order walk of the retrans queue */
	for (i = used_sacks - 1; i > 0; i--) {
1721 1722
		for (j = 0; j < i; j++) {
			if (after(sp[j].start_seq, sp[j + 1].start_seq)) {
1723
				swap(sp[j], sp[j + 1]);
1724

1725 1726
				/* Track where the first SACK block goes to */
				if (j == first_sack_index)
1727
					first_sack_index = j + 1;
1728 1729 1730 1731
			}
		}
	}

1732 1733
	state->mss_now = tcp_current_mss(sk);
	skb = NULL;
1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
	i = 0;

	if (!tp->sacked_out) {
		/* It's already past, so skip checking against it */
		cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
	} else {
		cache = tp->recv_sack_cache;
		/* Skip empty blocks in at head of the cache */
		while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq &&
		       !cache->end_seq)
			cache++;
1745 1746
	}

1747
	while (i < used_sacks) {
1748 1749
		u32 start_seq = sp[i].start_seq;
		u32 end_seq = sp[i].end_seq;
E
Eric Dumazet 已提交
1750
		bool dup_sack = (found_dup_sack && (i == first_sack_index));
1751
		struct tcp_sack_block *next_dup = NULL;
1752

1753 1754
		if (found_dup_sack && ((i + 1) == first_sack_index))
			next_dup = &sp[i + 1];
L
Linus Torvalds 已提交
1755

1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766
		/* Skip too early cached blocks */
		while (tcp_sack_cache_ok(tp, cache) &&
		       !before(start_seq, cache->end_seq))
			cache++;

		/* Can skip some work by looking recv_sack_cache? */
		if (tcp_sack_cache_ok(tp, cache) && !dup_sack &&
		    after(end_seq, cache->start_seq)) {

			/* Head todo? */
			if (before(start_seq, cache->start_seq)) {
1767
				skb = tcp_sacktag_skip(skb, sk, state,
1768
						       start_seq);
1769
				skb = tcp_sacktag_walk(skb, sk, next_dup,
1770
						       state,
1771 1772
						       start_seq,
						       cache->start_seq,
1773
						       dup_sack);
1774
			}
1775

1776
			/* Rest of the block already fully processed? */
1777
			if (!after(end_seq, cache->end_seq))
1778
				goto advance_sp;
1779

1780
			skb = tcp_maybe_skipping_dsack(skb, sk, next_dup,
1781
						       state,
1782
						       cache->end_seq);
1783

1784
			/* ...tail remains todo... */
1785
			if (tcp_highest_sack_seq(tp) == cache->end_seq) {
1786
				/* ...but better entrypoint exists! */
1787
				skb = tcp_highest_sack(sk);
1788
				if (!skb)
1789
					break;
1790 1791
				cache++;
				goto walk;
1792 1793
			}

1794
			skb = tcp_sacktag_skip(skb, sk, state, cache->end_seq);
1795 1796 1797 1798
			/* Check overlap against next cached too (past this one already) */
			cache++;
			continue;
		}
L
Linus Torvalds 已提交
1799

1800 1801
		if (!before(start_seq, tcp_highest_sack_seq(tp))) {
			skb = tcp_highest_sack(sk);
1802
			if (!skb)
1803
				break;
L
Linus Torvalds 已提交
1804
		}
1805
		skb = tcp_sacktag_skip(skb, sk, state, start_seq);
1806 1807

walk:
1808
		skb = tcp_sacktag_walk(skb, sk, next_dup, state,
1809
				       start_seq, end_seq, dup_sack);
1810

1811 1812
advance_sp:
		i++;
L
Linus Torvalds 已提交
1813 1814
	}

1815 1816 1817 1818 1819 1820 1821 1822
	/* Clear the head of the cache sack blocks so we can skip it next time */
	for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) {
		tp->recv_sack_cache[i].start_seq = 0;
		tp->recv_sack_cache[i].end_seq = 0;
	}
	for (j = 0; j < used_sacks; j++)
		tp->recv_sack_cache[i++] = sp[j];

1823 1824
	if (inet_csk(sk)->icsk_ca_state != TCP_CA_Loss || tp->undo_marker)
		tcp_check_sack_reordering(sk, state->reord, 0);
L
Linus Torvalds 已提交
1825

1826
	tcp_verify_left_out(tp);
1827 1828
out:

L
Linus Torvalds 已提交
1829
#if FASTRETRANS_DEBUG > 0
1830 1831 1832 1833
	WARN_ON((int)tp->sacked_out < 0);
	WARN_ON((int)tp->lost_out < 0);
	WARN_ON((int)tp->retrans_out < 0);
	WARN_ON((int)tcp_packets_in_flight(tp) < 0);
L
Linus Torvalds 已提交
1834
#endif
1835
	return state->flag;
L
Linus Torvalds 已提交
1836 1837
}

1838
/* Limits sacked_out so that sum with lost_out isn't ever larger than
E
Eric Dumazet 已提交
1839
 * packets_out. Returns false if sacked_out adjustement wasn't necessary.
1840
 */
E
Eric Dumazet 已提交
1841
static bool tcp_limit_reno_sacked(struct tcp_sock *tp)
1842 1843 1844 1845 1846 1847 1848 1849
{
	u32 holes;

	holes = max(tp->lost_out, 1U);
	holes = min(holes, tp->packets_out);

	if ((tp->sacked_out + holes) > tp->packets_out) {
		tp->sacked_out = tp->packets_out - holes;
E
Eric Dumazet 已提交
1850
		return true;
1851
	}
E
Eric Dumazet 已提交
1852
	return false;
1853 1854 1855 1856 1857 1858 1859 1860 1861
}

/* If we receive more dupacks than we expected counting segments
 * in assumption of absent reordering, interpret this as reordering.
 * The only another reason could be bug in receiver TCP.
 */
static void tcp_check_reno_reordering(struct sock *sk, const int addend)
{
	struct tcp_sock *tp = tcp_sk(sk);
1862 1863 1864 1865 1866 1867 1868

	if (!tcp_limit_reno_sacked(tp))
		return;

	tp->reordering = min_t(u32, tp->packets_out + addend,
			       sock_net(sk)->ipv4.sysctl_tcp_max_reordering);
	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER);
1869 1870 1871 1872 1873 1874 1875
}

/* Emulate SACKs for SACKless connection: account for a new dupack. */

static void tcp_add_reno_sack(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
Y
Yuchung Cheng 已提交
1876 1877
	u32 prior_sacked = tp->sacked_out;

1878 1879
	tp->sacked_out++;
	tcp_check_reno_reordering(sk, 0);
Y
Yuchung Cheng 已提交
1880 1881
	if (tp->sacked_out > prior_sacked)
		tp->delivered++; /* Some out-of-order packet is delivered */
1882
	tcp_verify_left_out(tp);
1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
}

/* Account for ACK, ACKing some data in Reno Recovery phase. */

static void tcp_remove_reno_sacks(struct sock *sk, int acked)
{
	struct tcp_sock *tp = tcp_sk(sk);

	if (acked > 0) {
		/* One ACK acked hole. The rest eat duplicate ACKs. */
Y
Yuchung Cheng 已提交
1893
		tp->delivered += max_t(int, acked - tp->sacked_out, 1);
1894
		if (acked - 1 >= tp->sacked_out)
1895 1896
			tp->sacked_out = 0;
		else
1897
			tp->sacked_out -= acked - 1;
1898 1899
	}
	tcp_check_reno_reordering(sk, acked);
1900
	tcp_verify_left_out(tp);
1901 1902 1903 1904 1905 1906 1907
}

static inline void tcp_reset_reno_sack(struct tcp_sock *tp)
{
	tp->sacked_out = 0;
}

Y
Yuchung Cheng 已提交
1908
void tcp_clear_retrans(struct tcp_sock *tp)
L
Linus Torvalds 已提交
1909 1910 1911 1912
{
	tp->retrans_out = 0;
	tp->lost_out = 0;
	tp->undo_marker = 0;
Y
Yuchung Cheng 已提交
1913
	tp->undo_retrans = -1;
Y
Yuchung Cheng 已提交
1914
	tp->sacked_out = 0;
L
Linus Torvalds 已提交
1915 1916
}

Y
Yuchung Cheng 已提交
1917
static inline void tcp_init_undo(struct tcp_sock *tp)
1918
{
Y
Yuchung Cheng 已提交
1919 1920 1921
	tp->undo_marker = tp->snd_una;
	/* Retransmission still in flight may cause DSACKs later. */
	tp->undo_retrans = tp->retrans_out ? : -1;
1922 1923
}

1924 1925 1926 1927 1928
static bool tcp_is_rack(const struct sock *sk)
{
	return sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_LOSS_DETECTION;
}

1929
/* If we detect SACK reneging, forget all SACK information
L
Linus Torvalds 已提交
1930 1931 1932
 * and reset tags completely, otherwise preserve SACKs. If receiver
 * dropped its ofo queue, we will know this due to reneging detection.
 */
1933 1934 1935
static void tcp_timeout_mark_lost(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
1936
	struct sk_buff *skb, *head;
1937 1938
	bool is_reneg;			/* is receiver reneging on SACKs? */

1939 1940
	head = tcp_rtx_queue_head(sk);
	is_reneg = head && (TCP_SKB_CB(head)->sacked & TCPCB_SACKED_ACKED);
1941 1942 1943 1944 1945 1946 1947 1948 1949
	if (is_reneg) {
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
		tp->sacked_out = 0;
		/* Mark SACK reneging until we recover from this loss event. */
		tp->is_sack_reneg = 1;
	} else if (tcp_is_reno(tp)) {
		tcp_reset_reno_sack(tp);
	}

1950
	skb = head;
1951 1952 1953
	skb_rbtree_walk_from(skb) {
		if (is_reneg)
			TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
1954 1955 1956
		else if (tcp_is_rack(sk) && skb != head &&
			 tcp_rack_skb_timeout(tp, skb, 0) > 0)
			continue; /* Don't mark recently sent ones lost yet */
1957 1958 1959 1960 1961 1962 1963
		tcp_mark_skb_lost(sk, skb);
	}
	tcp_verify_left_out(tp);
	tcp_clear_all_retrans_hints(tp);
}

/* Enter Loss state. */
1964
void tcp_enter_loss(struct sock *sk)
L
Linus Torvalds 已提交
1965
{
1966
	const struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
1967
	struct tcp_sock *tp = tcp_sk(sk);
1968
	struct net *net = sock_net(sk);
1969
	bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery;
L
Linus Torvalds 已提交
1970

1971 1972
	tcp_timeout_mark_lost(sk);

L
Linus Torvalds 已提交
1973
	/* Reduce ssthresh if it has not yet been made inside this window. */
Y
Yuchung Cheng 已提交
1974 1975
	if (icsk->icsk_ca_state <= TCP_CA_Disorder ||
	    !after(tp->high_seq, tp->snd_una) ||
1976 1977
	    (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) {
		tp->prior_ssthresh = tcp_current_ssthresh(sk);
1978
		tp->prior_cwnd = tp->snd_cwnd;
1979 1980
		tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
		tcp_ca_event(sk, CA_EVENT_LOSS);
Y
Yuchung Cheng 已提交
1981
		tcp_init_undo(tp);
L
Linus Torvalds 已提交
1982
	}
1983
	tp->snd_cwnd	   = tcp_packets_in_flight(tp) + 1;
L
Linus Torvalds 已提交
1984
	tp->snd_cwnd_cnt   = 0;
1985
	tp->snd_cwnd_stamp = tcp_jiffies32;
L
Linus Torvalds 已提交
1986

1987 1988 1989 1990
	/* Timeout in disordered state after receiving substantial DUPACKs
	 * suggests that the degree of reordering is over-estimated.
	 */
	if (icsk->icsk_ca_state <= TCP_CA_Disorder &&
1991
	    tp->sacked_out >= net->ipv4.sysctl_tcp_reordering)
1992
		tp->reordering = min_t(unsigned int, tp->reordering,
1993
				       net->ipv4.sysctl_tcp_reordering);
1994
	tcp_set_ca_state(sk, TCP_CA_Loss);
L
Linus Torvalds 已提交
1995
	tp->high_seq = tp->snd_nxt;
1996
	tcp_ecn_queue_cwr(tp);
Y
Yuchung Cheng 已提交
1997

1998 1999 2000
	/* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous
	 * loss recovery is underway except recurring timeout(s) on
	 * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing
Y
Yuchung Cheng 已提交
2001
	 */
E
Eric Dumazet 已提交
2002
	tp->frto = net->ipv4.sysctl_tcp_frto &&
2003 2004
		   (new_recovery || icsk->icsk_retransmits) &&
		   !inet_csk(sk)->icsk_mtup.probe_size;
L
Linus Torvalds 已提交
2005 2006
}

2007 2008 2009 2010
/* If ACK arrived pointing to a remembered SACK, it means that our
 * remembered SACKs do not reflect real state of receiver i.e.
 * receiver _host_ is heavily congested (or buggy).
 *
2011 2012 2013 2014 2015
 * To avoid big spurious retransmission bursts due to transient SACK
 * scoreboard oddities that look like reneging, we give the receiver a
 * little time (max(RTT/2, 10ms)) to send us some more ACKs that will
 * restore sanity to the SACK scoreboard. If the apparent reneging
 * persists until this RTO then we'll clear the SACK scoreboard.
2016
 */
E
Eric Dumazet 已提交
2017
static bool tcp_check_sack_reneging(struct sock *sk, int flag)
L
Linus Torvalds 已提交
2018
{
2019
	if (flag & FLAG_SACK_RENEGING) {
2020 2021 2022
		struct tcp_sock *tp = tcp_sk(sk);
		unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4),
					  msecs_to_jiffies(10));
L
Linus Torvalds 已提交
2023

2024
		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2025
					  delay, TCP_RTO_MAX);
E
Eric Dumazet 已提交
2026
		return true;
L
Linus Torvalds 已提交
2027
	}
E
Eric Dumazet 已提交
2028
	return false;
L
Linus Torvalds 已提交
2029 2030
}

2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041
/* Heurestics to calculate number of duplicate ACKs. There's no dupACKs
 * counter when SACK is enabled (without SACK, sacked_out is used for
 * that purpose).
 *
 * With reordering, holes may still be in flight, so RFC3517 recovery
 * uses pure sacked_out (total number of SACKed segments) even though
 * it violates the RFC that uses duplicate ACKs, often these are equal
 * but when e.g. out-of-window ACKs or packet duplication occurs,
 * they differ. Since neither occurs due to loss, TCP should really
 * ignore them.
 */
2042
static inline int tcp_dupack_heuristics(const struct tcp_sock *tp)
2043
{
Y
Yuchung Cheng 已提交
2044
	return tp->sacked_out + 1;
2045 2046
}

Y
Yuchung Cheng 已提交
2047
/* Linux NewReno/SACK/ECN state machine.
L
Linus Torvalds 已提交
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093
 * --------------------------------------
 *
 * "Open"	Normal state, no dubious events, fast path.
 * "Disorder"   In all the respects it is "Open",
 *		but requires a bit more attention. It is entered when
 *		we see some SACKs or dupacks. It is split of "Open"
 *		mainly to move some processing from fast path to slow one.
 * "CWR"	CWND was reduced due to some Congestion Notification event.
 *		It can be ECN, ICMP source quench, local device congestion.
 * "Recovery"	CWND was reduced, we are fast-retransmitting.
 * "Loss"	CWND was reduced due to RTO timeout or SACK reneging.
 *
 * tcp_fastretrans_alert() is entered:
 * - each incoming ACK, if state is not "Open"
 * - when arrived ACK is unusual, namely:
 *	* SACK
 *	* Duplicate ACK.
 *	* ECN ECE.
 *
 * Counting packets in flight is pretty simple.
 *
 *	in_flight = packets_out - left_out + retrans_out
 *
 *	packets_out is SND.NXT-SND.UNA counted in packets.
 *
 *	retrans_out is number of retransmitted segments.
 *
 *	left_out is number of segments left network, but not ACKed yet.
 *
 *		left_out = sacked_out + lost_out
 *
 *     sacked_out: Packets, which arrived to receiver out of order
 *		   and hence not ACKed. With SACKs this number is simply
 *		   amount of SACKed data. Even without SACKs
 *		   it is easy to give pretty reliable estimate of this number,
 *		   counting duplicate ACKs.
 *
 *       lost_out: Packets lost by network. TCP has no explicit
 *		   "loss notification" feedback from network (for now).
 *		   It means that this number can be only _guessed_.
 *		   Actually, it is the heuristics to predict lossage that
 *		   distinguishes different algorithms.
 *
 *	F.e. after RTO, when all the queue is considered as lost,
 *	lost_out = packets_out and in_flight = retrans_out.
 *
2094
 *		Essentially, we have now a few algorithms detecting
L
Linus Torvalds 已提交
2095 2096
 *		lost packets.
 *
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114
 *		If the receiver supports SACK:
 *
 *		RFC6675/3517: It is the conventional algorithm. A packet is
 *		considered lost if the number of higher sequence packets
 *		SACKed is greater than or equal the DUPACK thoreshold
 *		(reordering). This is implemented in tcp_mark_head_lost and
 *		tcp_update_scoreboard.
 *
 *		RACK (draft-ietf-tcpm-rack-01): it is a newer algorithm
 *		(2017-) that checks timing instead of counting DUPACKs.
 *		Essentially a packet is considered lost if it's not S/ACKed
 *		after RTT + reordering_window, where both metrics are
 *		dynamically measured and adjusted. This is implemented in
 *		tcp_rack_mark_lost.
 *
 *		If the receiver does not support SACK:
 *
 *		NewReno (RFC6582): in Recovery we assume that one segment
L
Linus Torvalds 已提交
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143
 *		is lost (classic Reno). While we are in Recovery and
 *		a partial ACK arrives, we assume that one more packet
 *		is lost (NewReno). This heuristics are the same in NewReno
 *		and SACK.
 *
 * Really tricky (and requiring careful tuning) part of algorithm
 * is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue().
 * The first determines the moment _when_ we should reduce CWND and,
 * hence, slow down forward transmission. In fact, it determines the moment
 * when we decide that hole is caused by loss, rather than by a reorder.
 *
 * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill
 * holes, caused by lost packets.
 *
 * And the most logically complicated part of algorithm is undo
 * heuristics. We detect false retransmits due to both too early
 * fast retransmit (reordering) and underestimated RTO, analyzing
 * timestamps and D-SACKs. When we detect that some segments were
 * retransmitted by mistake and CWND reduction was wrong, we undo
 * window reduction and abort recovery phase. This logic is hidden
 * inside several functions named tcp_try_undo_<something>.
 */

/* This function decides, when we should leave Disordered state
 * and enter Recovery phase, reducing congestion window.
 *
 * Main question: may we further continue forward transmission
 * with the same cwnd?
 */
E
Eric Dumazet 已提交
2144
static bool tcp_time_to_recover(struct sock *sk, int flag)
L
Linus Torvalds 已提交
2145
{
2146
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
2147 2148 2149

	/* Trick#1: The loss is proven. */
	if (tp->lost_out)
E
Eric Dumazet 已提交
2150
		return true;
L
Linus Torvalds 已提交
2151 2152

	/* Not-A-Trick#2 : Classic rule... */
2153
	if (!tcp_is_rack(sk) && tcp_dupack_heuristics(tp) > tp->reordering)
E
Eric Dumazet 已提交
2154
		return true;
L
Linus Torvalds 已提交
2155

E
Eric Dumazet 已提交
2156
	return false;
L
Linus Torvalds 已提交
2157 2158
}

2159
/* Detect loss in event "A" above by marking head of queue up as lost.
Y
Yuchung Cheng 已提交
2160
 * For non-SACK(Reno) senders, the first "packets" number of segments
2161 2162 2163
 * are considered lost. For RFC3517 SACK, a segment is considered lost if it
 * has at least tp->reordering SACKed seqments above it; "packets" refers to
 * the maximum SACKed segments to pass before reaching this limit.
2164
 */
I
Ilpo Järvinen 已提交
2165
static void tcp_mark_head_lost(struct sock *sk, int packets, int mark_head)
L
Linus Torvalds 已提交
2166
{
2167
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
2168
	struct sk_buff *skb;
2169
	int cnt, oldcnt, lost;
2170
	unsigned int mss;
2171 2172
	/* Use SACK to deduce losses of new sequences sent during recovery */
	const u32 loss_high = tcp_is_sack(tp) ?  tp->snd_nxt : tp->high_seq;
L
Linus Torvalds 已提交
2173

2174
	WARN_ON(packets > tp->packets_out);
2175 2176
	skb = tp->lost_skb_hint;
	if (skb) {
I
Ilpo Järvinen 已提交
2177
		/* Head already handled? */
2178
		if (mark_head && after(TCP_SKB_CB(skb)->seq, tp->snd_una))
I
Ilpo Järvinen 已提交
2179
			return;
2180
		cnt = tp->lost_cnt_hint;
2181
	} else {
2182
		skb = tcp_rtx_queue_head(sk);
2183 2184
		cnt = 0;
	}
L
Linus Torvalds 已提交
2185

2186
	skb_rbtree_walk_from(skb) {
2187 2188 2189 2190
		/* TODO: do this better */
		/* this is not the most efficient way to do this... */
		tp->lost_skb_hint = skb;
		tp->lost_cnt_hint = cnt;
2191

2192
		if (after(TCP_SKB_CB(skb)->end_seq, loss_high))
2193 2194 2195
			break;

		oldcnt = cnt;
Y
Yuchung Cheng 已提交
2196
		if (tcp_is_reno(tp) ||
2197 2198 2199
		    (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
			cnt += tcp_skb_pcount(skb);

2200
		if (cnt > packets) {
Y
Yuchung Cheng 已提交
2201
			if (tcp_is_sack(tp) ||
2202
			    (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) ||
2203
			    (oldcnt >= packets))
2204 2205
				break;

2206
			mss = tcp_skb_mss(skb);
2207 2208 2209
			/* If needed, chop off the prefix to mark as lost. */
			lost = (packets - oldcnt) * mss;
			if (lost < skb->len &&
2210 2211
			    tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
					 lost, mss, GFP_ATOMIC) < 0)
2212 2213 2214 2215
				break;
			cnt = packets;
		}

2216
		tcp_skb_mark_lost(tp, skb);
I
Ilpo Järvinen 已提交
2217 2218 2219

		if (mark_head)
			break;
L
Linus Torvalds 已提交
2220
	}
2221
	tcp_verify_left_out(tp);
L
Linus Torvalds 已提交
2222 2223 2224 2225
}

/* Account newly detected lost packet(s) */

2226
static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit)
L
Linus Torvalds 已提交
2227
{
2228 2229
	struct tcp_sock *tp = tcp_sk(sk);

2230
	if (tcp_is_sack(tp)) {
2231
		int sacked_upto = tp->sacked_out - tp->reordering;
I
Ilpo Järvinen 已提交
2232 2233 2234 2235
		if (sacked_upto >= 0)
			tcp_mark_head_lost(sk, sacked_upto, 0);
		else if (fast_rexmit)
			tcp_mark_head_lost(sk, 1, 1);
L
Linus Torvalds 已提交
2236 2237 2238
	}
}

2239 2240 2241 2242 2243 2244
static bool tcp_tsopt_ecr_before(const struct tcp_sock *tp, u32 when)
{
	return tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
	       before(tp->rx_opt.rcv_tsecr, when);
}

2245 2246 2247 2248 2249 2250 2251 2252 2253 2254
/* skb is spurious retransmitted if the returned timestamp echo
 * reply is prior to the skb transmission time
 */
static bool tcp_skb_spurious_retrans(const struct tcp_sock *tp,
				     const struct sk_buff *skb)
{
	return (TCP_SKB_CB(skb)->sacked & TCPCB_RETRANS) &&
	       tcp_tsopt_ecr_before(tp, tcp_skb_timestamp(skb));
}

L
Linus Torvalds 已提交
2255 2256 2257
/* Nothing was retransmitted or returned timestamp is less
 * than timestamp of the first retransmission.
 */
2258
static inline bool tcp_packet_delayed(const struct tcp_sock *tp)
L
Linus Torvalds 已提交
2259 2260
{
	return !tp->retrans_stamp ||
2261
	       tcp_tsopt_ecr_before(tp, tp->retrans_stamp);
L
Linus Torvalds 已提交
2262 2263 2264 2265
}

/* Undo procedures. */

2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287
/* We can clear retrans_stamp when there are no retransmissions in the
 * window. It would seem that it is trivially available for us in
 * tp->retrans_out, however, that kind of assumptions doesn't consider
 * what will happen if errors occur when sending retransmission for the
 * second time. ...It could the that such segment has only
 * TCPCB_EVER_RETRANS set at the present time. It seems that checking
 * the head skb is enough except for some reneging corner cases that
 * are not worth the effort.
 *
 * Main reason for all this complexity is the fact that connection dying
 * time now depends on the validity of the retrans_stamp, in particular,
 * that successive retransmissions of a segment must not advance
 * retrans_stamp under any conditions.
 */
static bool tcp_any_retrans_done(const struct sock *sk)
{
	const struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb;

	if (tp->retrans_out)
		return true;

2288
	skb = tcp_rtx_queue_head(sk);
2289 2290 2291 2292 2293 2294
	if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS))
		return true;

	return false;
}

2295
static void DBGUNDO(struct sock *sk, const char *msg)
L
Linus Torvalds 已提交
2296
{
J
Joe Perches 已提交
2297
#if FASTRETRANS_DEBUG > 1
2298
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
2299
	struct inet_sock *inet = inet_sk(sk);
2300

2301
	if (sk->sk_family == AF_INET) {
2302 2303 2304 2305 2306 2307
		pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n",
			 msg,
			 &inet->inet_daddr, ntohs(inet->inet_dport),
			 tp->snd_cwnd, tcp_left_out(tp),
			 tp->snd_ssthresh, tp->prior_ssthresh,
			 tp->packets_out);
2308
	}
E
Eric Dumazet 已提交
2309
#if IS_ENABLED(CONFIG_IPV6)
2310
	else if (sk->sk_family == AF_INET6) {
2311 2312
		pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n",
			 msg,
2313
			 &sk->sk_v6_daddr, ntohs(inet->inet_dport),
2314 2315 2316
			 tp->snd_cwnd, tcp_left_out(tp),
			 tp->snd_ssthresh, tp->prior_ssthresh,
			 tp->packets_out);
2317 2318
	}
#endif
L
Linus Torvalds 已提交
2319
#endif
J
Joe Perches 已提交
2320
}
L
Linus Torvalds 已提交
2321

2322
static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss)
L
Linus Torvalds 已提交
2323
{
2324 2325
	struct tcp_sock *tp = tcp_sk(sk);

Y
Yuchung Cheng 已提交
2326 2327 2328
	if (unmark_loss) {
		struct sk_buff *skb;

2329
		skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
Y
Yuchung Cheng 已提交
2330 2331 2332 2333 2334 2335
			TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST;
		}
		tp->lost_out = 0;
		tcp_clear_all_retrans_hints(tp);
	}

L
Linus Torvalds 已提交
2336
	if (tp->prior_ssthresh) {
2337 2338
		const struct inet_connection_sock *icsk = inet_csk(sk);

2339
		tp->snd_cwnd = icsk->icsk_ca_ops->undo_cwnd(sk);
L
Linus Torvalds 已提交
2340

2341
		if (tp->prior_ssthresh > tp->snd_ssthresh) {
L
Linus Torvalds 已提交
2342
			tp->snd_ssthresh = tp->prior_ssthresh;
2343
			tcp_ecn_withdraw_cwr(tp);
L
Linus Torvalds 已提交
2344 2345
		}
	}
2346
	tp->snd_cwnd_stamp = tcp_jiffies32;
2347
	tp->undo_marker = 0;
2348
	tp->rack.advanced = 1; /* Force RACK to re-exam losses */
L
Linus Torvalds 已提交
2349 2350
}

2351
static inline bool tcp_may_undo(const struct tcp_sock *tp)
L
Linus Torvalds 已提交
2352
{
2353
	return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp));
L
Linus Torvalds 已提交
2354 2355 2356
}

/* People celebrate: "We love our President!" */
E
Eric Dumazet 已提交
2357
static bool tcp_try_undo_recovery(struct sock *sk)
L
Linus Torvalds 已提交
2358
{
2359 2360
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
2361
	if (tcp_may_undo(tp)) {
2362 2363
		int mib_idx;

L
Linus Torvalds 已提交
2364 2365 2366
		/* Happy end! We did not retransmit anything
		 * or our original transmission succeeded.
		 */
2367
		DBGUNDO(sk, inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans");
2368
		tcp_undo_cwnd_reduction(sk, false);
2369
		if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
2370
			mib_idx = LINUX_MIB_TCPLOSSUNDO;
L
Linus Torvalds 已提交
2371
		else
2372 2373
			mib_idx = LINUX_MIB_TCPFULLUNDO;

2374
		NET_INC_STATS(sock_net(sk), mib_idx);
2375 2376
	} else if (tp->rack.reo_wnd_persist) {
		tp->rack.reo_wnd_persist--;
L
Linus Torvalds 已提交
2377
	}
2378
	if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) {
L
Linus Torvalds 已提交
2379 2380 2381
		/* Hold old state until something *above* high_seq
		 * is ACKed. For Reno it is MUST to prevent false
		 * fast retransmits (RFC2582). SACK TCP is safe. */
2382 2383
		if (!tcp_any_retrans_done(sk))
			tp->retrans_stamp = 0;
E
Eric Dumazet 已提交
2384
		return true;
L
Linus Torvalds 已提交
2385
	}
2386
	tcp_set_ca_state(sk, TCP_CA_Open);
2387
	tp->is_sack_reneg = 0;
E
Eric Dumazet 已提交
2388
	return false;
L
Linus Torvalds 已提交
2389 2390 2391
}

/* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */
2392
static bool tcp_try_undo_dsack(struct sock *sk)
L
Linus Torvalds 已提交
2393
{
2394 2395
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
2396
	if (tp->undo_marker && !tp->undo_retrans) {
2397 2398
		tp->rack.reo_wnd_persist = min(TCP_RACK_RECOVERY_THRESH,
					       tp->rack.reo_wnd_persist + 1);
2399
		DBGUNDO(sk, "D-SACK");
2400
		tcp_undo_cwnd_reduction(sk, false);
2401
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKUNDO);
2402
		return true;
L
Linus Torvalds 已提交
2403
	}
2404
	return false;
L
Linus Torvalds 已提交
2405 2406
}

Y
Yuchung Cheng 已提交
2407 2408
/* Undo during loss recovery after partial ACK or using F-RTO. */
static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo)
L
Linus Torvalds 已提交
2409
{
2410 2411
	struct tcp_sock *tp = tcp_sk(sk);

Y
Yuchung Cheng 已提交
2412
	if (frto_undo || tcp_may_undo(tp)) {
2413
		tcp_undo_cwnd_reduction(sk, true);
2414

2415
		DBGUNDO(sk, "partial loss");
2416
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSUNDO);
Y
Yuchung Cheng 已提交
2417
		if (frto_undo)
2418
			NET_INC_STATS(sock_net(sk),
2419
					LINUX_MIB_TCPSPURIOUSRTOS);
2420
		inet_csk(sk)->icsk_retransmits = 0;
2421
		if (frto_undo || tcp_is_sack(tp)) {
2422
			tcp_set_ca_state(sk, TCP_CA_Open);
2423 2424
			tp->is_sack_reneg = 0;
		}
E
Eric Dumazet 已提交
2425
		return true;
L
Linus Torvalds 已提交
2426
	}
E
Eric Dumazet 已提交
2427
	return false;
L
Linus Torvalds 已提交
2428 2429
}

2430
/* The cwnd reduction in CWR and Recovery uses the PRR algorithm in RFC 6937.
2431 2432 2433 2434
 * It computes the number of packets to send (sndcnt) based on packets newly
 * delivered:
 *   1) If the packets in flight is larger than ssthresh, PRR spreads the
 *	cwnd reductions across a full RTT.
2435 2436 2437
 *   2) Otherwise PRR uses packet conservation to send as much as delivered.
 *      But when the retransmits are acked without further losses, PRR
 *      slow starts cwnd up to ssthresh to speed up the recovery.
2438
 */
2439
static void tcp_init_cwnd_reduction(struct sock *sk)
2440 2441 2442 2443
{
	struct tcp_sock *tp = tcp_sk(sk);

	tp->high_seq = tp->snd_nxt;
N
Nandita Dukkipati 已提交
2444
	tp->tlp_high_seq = 0;
2445 2446 2447 2448
	tp->snd_cwnd_cnt = 0;
	tp->prior_cwnd = tp->snd_cwnd;
	tp->prr_delivered = 0;
	tp->prr_out = 0;
2449
	tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk);
2450
	tcp_ecn_queue_cwr(tp);
2451 2452
}

2453
void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag)
2454 2455 2456 2457 2458
{
	struct tcp_sock *tp = tcp_sk(sk);
	int sndcnt = 0;
	int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp);

2459 2460 2461
	if (newly_acked_sacked <= 0 || WARN_ON_ONCE(!tp->prior_cwnd))
		return;

2462
	tp->prr_delivered += newly_acked_sacked;
2463
	if (delta < 0) {
2464 2465 2466
		u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered +
			       tp->prior_cwnd - 1;
		sndcnt = div_u64(dividend, tp->prior_cwnd) - tp->prr_out;
2467 2468
	} else if ((flag & FLAG_RETRANS_DATA_ACKED) &&
		   !(flag & FLAG_LOST_RETRANS)) {
2469 2470 2471
		sndcnt = min_t(int, delta,
			       max_t(int, tp->prr_delivered - tp->prr_out,
				     newly_acked_sacked) + 1);
2472 2473
	} else {
		sndcnt = min(delta, newly_acked_sacked);
2474
	}
2475 2476
	/* Force a fast retransmit upon entering fast recovery */
	sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1));
2477 2478 2479
	tp->snd_cwnd = tcp_packets_in_flight(tp) + sndcnt;
}

2480
static inline void tcp_end_cwnd_reduction(struct sock *sk)
L
Linus Torvalds 已提交
2481
{
2482
	struct tcp_sock *tp = tcp_sk(sk);
2483

2484 2485 2486
	if (inet_csk(sk)->icsk_ca_ops->cong_control)
		return;

2487
	/* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
2488 2489
	if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH &&
	    (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || tp->undo_marker)) {
2490
		tp->snd_cwnd = tp->snd_ssthresh;
2491
		tp->snd_cwnd_stamp = tcp_jiffies32;
2492
	}
2493
	tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR);
L
Linus Torvalds 已提交
2494 2495
}

2496
/* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */
2497
void tcp_enter_cwr(struct sock *sk)
Y
Yuchung Cheng 已提交
2498 2499 2500 2501
{
	struct tcp_sock *tp = tcp_sk(sk);

	tp->prior_ssthresh = 0;
2502
	if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) {
Y
Yuchung Cheng 已提交
2503
		tp->undo_marker = 0;
2504
		tcp_init_cwnd_reduction(sk);
Y
Yuchung Cheng 已提交
2505 2506 2507
		tcp_set_ca_state(sk, TCP_CA_CWR);
	}
}
2508
EXPORT_SYMBOL(tcp_enter_cwr);
Y
Yuchung Cheng 已提交
2509

2510 2511 2512 2513 2514
static void tcp_try_keep_open(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	int state = TCP_CA_Open;

2515
	if (tcp_left_out(tp) || tcp_any_retrans_done(sk))
2516 2517 2518 2519 2520 2521 2522 2523
		state = TCP_CA_Disorder;

	if (inet_csk(sk)->icsk_ca_state != state) {
		tcp_set_ca_state(sk, state);
		tp->high_seq = tp->snd_nxt;
	}
}

2524
static void tcp_try_to_open(struct sock *sk, int flag)
L
Linus Torvalds 已提交
2525
{
2526 2527
	struct tcp_sock *tp = tcp_sk(sk);

2528 2529
	tcp_verify_left_out(tp);

Y
Yuchung Cheng 已提交
2530
	if (!tcp_any_retrans_done(sk))
L
Linus Torvalds 已提交
2531 2532
		tp->retrans_stamp = 0;

2533
	if (flag & FLAG_ECE)
2534
		tcp_enter_cwr(sk);
L
Linus Torvalds 已提交
2535

2536
	if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) {
2537
		tcp_try_keep_open(sk);
L
Linus Torvalds 已提交
2538 2539 2540
	}
}

J
John Heffner 已提交
2541 2542 2543 2544 2545 2546
static void tcp_mtup_probe_failed(struct sock *sk)
{
	struct inet_connection_sock *icsk = inet_csk(sk);

	icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1;
	icsk->icsk_mtup.probe_size = 0;
2547
	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPFAIL);
J
John Heffner 已提交
2548 2549
}

2550
static void tcp_mtup_probe_success(struct sock *sk)
J
John Heffner 已提交
2551 2552 2553 2554 2555 2556 2557 2558 2559 2560
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct inet_connection_sock *icsk = inet_csk(sk);

	/* FIXME: breaks with very large cwnd */
	tp->prior_ssthresh = tcp_current_ssthresh(sk);
	tp->snd_cwnd = tp->snd_cwnd *
		       tcp_mss_to_mtu(sk, tp->mss_cache) /
		       icsk->icsk_mtup.probe_size;
	tp->snd_cwnd_cnt = 0;
2561
	tp->snd_cwnd_stamp = tcp_jiffies32;
2562
	tp->snd_ssthresh = tcp_current_ssthresh(sk);
J
John Heffner 已提交
2563 2564 2565 2566

	icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size;
	icsk->icsk_mtup.probe_size = 0;
	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
2567
	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPSUCCESS);
J
John Heffner 已提交
2568 2569
}

2570 2571 2572 2573 2574 2575 2576 2577 2578
/* Do a simple retransmit without using the backoff mechanisms in
 * tcp_timer. This is used for path mtu discovery.
 * The socket is already locked here.
 */
void tcp_simple_retransmit(struct sock *sk)
{
	const struct inet_connection_sock *icsk = inet_csk(sk);
	struct tcp_sock *tp = tcp_sk(sk);
	struct sk_buff *skb;
I
Ilpo Järvinen 已提交
2579
	unsigned int mss = tcp_current_mss(sk);
2580

2581
	skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
2582
		if (tcp_skb_seglen(skb) > mss &&
2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
		    !(TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)) {
			if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
				TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
				tp->retrans_out -= tcp_skb_pcount(skb);
			}
			tcp_skb_mark_lost_uncond_verify(tp, skb);
		}
	}

	tcp_clear_retrans_hints_partial(tp);

2594
	if (!tp->lost_out)
2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615
		return;

	if (tcp_is_reno(tp))
		tcp_limit_reno_sacked(tp);

	tcp_verify_left_out(tp);

	/* Don't muck with the congestion window here.
	 * Reason is that we do not increase amount of _data_
	 * in network, but units changed and effective
	 * cwnd/ssthresh really reduced now.
	 */
	if (icsk->icsk_ca_state != TCP_CA_Loss) {
		tp->high_seq = tp->snd_nxt;
		tp->snd_ssthresh = tcp_current_ssthresh(sk);
		tp->prior_ssthresh = 0;
		tp->undo_marker = 0;
		tcp_set_ca_state(sk, TCP_CA_Loss);
	}
	tcp_xmit_retransmit_queue(sk);
}
E
Eric Dumazet 已提交
2616
EXPORT_SYMBOL(tcp_simple_retransmit);
2617

2618
void tcp_enter_recovery(struct sock *sk, bool ece_ack)
2619 2620 2621 2622 2623 2624 2625 2626 2627
{
	struct tcp_sock *tp = tcp_sk(sk);
	int mib_idx;

	if (tcp_is_reno(tp))
		mib_idx = LINUX_MIB_TCPRENORECOVERY;
	else
		mib_idx = LINUX_MIB_TCPSACKRECOVERY;

2628
	NET_INC_STATS(sock_net(sk), mib_idx);
2629 2630

	tp->prior_ssthresh = 0;
Y
Yuchung Cheng 已提交
2631
	tcp_init_undo(tp);
2632

2633
	if (!tcp_in_cwnd_reduction(sk)) {
2634 2635
		if (!ece_ack)
			tp->prior_ssthresh = tcp_current_ssthresh(sk);
2636
		tcp_init_cwnd_reduction(sk);
2637 2638 2639 2640
	}
	tcp_set_ca_state(sk, TCP_CA_Recovery);
}

2641 2642 2643
/* Process an ACK in CA_Loss state. Move to CA_Open if lost data are
 * recovered or spurious. Otherwise retransmits more on partial ACKs.
 */
2644 2645
static void tcp_process_loss(struct sock *sk, int flag, bool is_dupack,
			     int *rexmit)
2646 2647
{
	struct tcp_sock *tp = tcp_sk(sk);
Y
Yuchung Cheng 已提交
2648
	bool recovered = !before(tp->snd_una, tp->high_seq);
2649

2650 2651 2652 2653
	if ((flag & FLAG_SND_UNA_ADVANCED) &&
	    tcp_try_undo_loss(sk, false))
		return;

2654
	if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */
2655 2656 2657 2658 2659 2660 2661
		/* Step 3.b. A timeout is spurious if not all data are
		 * lost, i.e., never-retransmitted data are (s)acked.
		 */
		if ((flag & FLAG_ORIG_SACK_ACKED) &&
		    tcp_try_undo_loss(sk, true))
			return;

2662 2663 2664
		if (after(tp->snd_nxt, tp->high_seq)) {
			if (flag & FLAG_DATA_SACKED || is_dupack)
				tp->frto = 0; /* Step 3.a. loss was real */
Y
Yuchung Cheng 已提交
2665 2666
		} else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) {
			tp->high_seq = tp->snd_nxt;
2667 2668 2669 2670
			/* Step 2.b. Try send new data (but deferred until cwnd
			 * is updated in tcp_ack()). Otherwise fall back to
			 * the conventional recovery.
			 */
2671
			if (!tcp_write_queue_empty(sk) &&
2672 2673 2674 2675
			    after(tcp_wnd_end(tp), tp->snd_nxt)) {
				*rexmit = REXMIT_NEW;
				return;
			}
Y
Yuchung Cheng 已提交
2676 2677 2678 2679 2680 2681
			tp->frto = 0;
		}
	}

	if (recovered) {
		/* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */
2682 2683 2684
		tcp_try_undo_recovery(sk);
		return;
	}
Y
Yuchung Cheng 已提交
2685 2686 2687 2688 2689 2690 2691 2692 2693
	if (tcp_is_reno(tp)) {
		/* A Reno DUPACK means new data in F-RTO step 2.b above are
		 * delivered. Lower inflight to clock out (re)tranmissions.
		 */
		if (after(tp->snd_nxt, tp->high_seq) && is_dupack)
			tcp_add_reno_sack(sk);
		else if (flag & FLAG_SND_UNA_ADVANCED)
			tcp_reset_reno_sack(tp);
	}
2694
	*rexmit = REXMIT_LOST;
2695 2696
}

Y
Yuchung Cheng 已提交
2697
/* Undo during fast recovery after partial ACK. */
2698
static bool tcp_try_undo_partial(struct sock *sk, u32 prior_snd_una)
Y
Yuchung Cheng 已提交
2699 2700 2701
{
	struct tcp_sock *tp = tcp_sk(sk);

2702
	if (tp->undo_marker && tcp_packet_delayed(tp)) {
Y
Yuchung Cheng 已提交
2703
		/* Plain luck! Hole if filled with delayed
2704
		 * packet, rather than with a retransmit. Check reordering.
Y
Yuchung Cheng 已提交
2705
		 */
2706
		tcp_check_sack_reordering(sk, prior_snd_una, 1);
2707 2708 2709 2710 2711 2712

		/* We are getting evidence that the reordering degree is higher
		 * than we realized. If there are no retransmits out then we
		 * can undo. Otherwise we clock out new packets but do not
		 * mark more packets lost or retransmit more.
		 */
2713
		if (tp->retrans_out)
2714 2715
			return true;

Y
Yuchung Cheng 已提交
2716 2717 2718
		if (!tcp_any_retrans_done(sk))
			tp->retrans_stamp = 0;

2719 2720
		DBGUNDO(sk, "partial recovery");
		tcp_undo_cwnd_reduction(sk, true);
2721
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO);
2722 2723
		tcp_try_keep_open(sk);
		return true;
Y
Yuchung Cheng 已提交
2724
	}
2725
	return false;
Y
Yuchung Cheng 已提交
2726 2727
}

2728
static void tcp_identify_packet_loss(struct sock *sk, int *ack_flag)
2729 2730 2731
{
	struct tcp_sock *tp = tcp_sk(sk);

2732 2733 2734 2735 2736 2737
	if (tcp_rtx_queue_empty(sk))
		return;

	if (unlikely(tcp_is_reno(tp))) {
		tcp_newreno_mark_lost(sk, *ack_flag & FLAG_SND_UNA_ADVANCED);
	} else if (tcp_is_rack(sk)) {
2738 2739
		u32 prior_retrans = tp->retrans_out;

2740
		tcp_rack_mark_lost(sk);
2741 2742 2743 2744 2745
		if (prior_retrans > tp->retrans_out)
			*ack_flag |= FLAG_LOST_RETRANS;
	}
}

2746 2747 2748 2749 2750 2751 2752 2753
static bool tcp_force_fast_retransmit(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);

	return after(tcp_highest_sack_seq(tp),
		     tp->snd_una + tp->reordering * tp->mss_cache);
}

L
Linus Torvalds 已提交
2754 2755 2756 2757 2758
/* Process an event, which can update packets-in-flight not trivially.
 * Main goal of this function is to calculate new estimate for left_out,
 * taking into account both packets sitting in receiver's buffer and
 * packets lost by network.
 *
2759 2760 2761
 * Besides that it updates the congestion state when packet loss or ECN
 * is detected. But it does not reduce the cwnd, it is done by the
 * congestion control later.
L
Linus Torvalds 已提交
2762 2763 2764 2765
 *
 * It does _not_ decide what to send, it is made in function
 * tcp_xmit_retransmit_queue().
 */
2766
static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
2767
				  bool is_dupack, int *ack_flag, int *rexmit)
L
Linus Torvalds 已提交
2768
{
2769
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
2770
	struct tcp_sock *tp = tcp_sk(sk);
2771
	int fast_rexmit = 0, flag = *ack_flag;
Y
Yuchung Cheng 已提交
2772
	bool do_lost = is_dupack || ((flag & FLAG_DATA_SACKED) &&
2773
				     tcp_force_fast_retransmit(sk));
L
Linus Torvalds 已提交
2774

2775
	if (!tp->packets_out && tp->sacked_out)
L
Linus Torvalds 已提交
2776 2777
		tp->sacked_out = 0;

2778
	/* Now state machine starts.
L
Linus Torvalds 已提交
2779
	 * A. ECE, hence prohibit cwnd undoing, the reduction is required. */
2780
	if (flag & FLAG_ECE)
L
Linus Torvalds 已提交
2781 2782 2783
		tp->prior_ssthresh = 0;

	/* B. In all the states check for reneging SACKs. */
2784
	if (tcp_check_sack_reneging(sk, flag))
L
Linus Torvalds 已提交
2785 2786
		return;

2787
	/* C. Check consistency of the current state. */
2788
	tcp_verify_left_out(tp);
L
Linus Torvalds 已提交
2789

2790
	/* D. Check state exit conditions. State can be terminated
L
Linus Torvalds 已提交
2791
	 *    when high_seq is ACKed. */
2792
	if (icsk->icsk_ca_state == TCP_CA_Open) {
2793
		WARN_ON(tp->retrans_out != 0);
L
Linus Torvalds 已提交
2794 2795
		tp->retrans_stamp = 0;
	} else if (!before(tp->snd_una, tp->high_seq)) {
2796
		switch (icsk->icsk_ca_state) {
L
Linus Torvalds 已提交
2797 2798 2799 2800
		case TCP_CA_CWR:
			/* CWR is to be held something *above* high_seq
			 * is ACKed for CWR bit to reach receiver. */
			if (tp->snd_una != tp->high_seq) {
2801
				tcp_end_cwnd_reduction(sk);
2802
				tcp_set_ca_state(sk, TCP_CA_Open);
L
Linus Torvalds 已提交
2803 2804 2805 2806
			}
			break;

		case TCP_CA_Recovery:
2807
			if (tcp_is_reno(tp))
L
Linus Torvalds 已提交
2808
				tcp_reset_reno_sack(tp);
2809
			if (tcp_try_undo_recovery(sk))
L
Linus Torvalds 已提交
2810
				return;
2811
			tcp_end_cwnd_reduction(sk);
L
Linus Torvalds 已提交
2812 2813 2814 2815
			break;
		}
	}

2816
	/* E. Process state. */
2817
	switch (icsk->icsk_ca_state) {
L
Linus Torvalds 已提交
2818
	case TCP_CA_Recovery:
2819
		if (!(flag & FLAG_SND_UNA_ADVANCED)) {
2820
			if (tcp_is_reno(tp) && is_dupack)
2821
				tcp_add_reno_sack(sk);
2822
		} else {
2823
			if (tcp_try_undo_partial(sk, prior_snd_una))
2824 2825 2826
				return;
			/* Partial ACK arrived. Force fast retransmit. */
			do_lost = tcp_is_reno(tp) ||
2827
				  tcp_force_fast_retransmit(sk);
2828
		}
2829 2830 2831 2832
		if (tcp_try_undo_dsack(sk)) {
			tcp_try_keep_open(sk);
			return;
		}
2833
		tcp_identify_packet_loss(sk, ack_flag);
L
Linus Torvalds 已提交
2834 2835
		break;
	case TCP_CA_Loss:
2836
		tcp_process_loss(sk, flag, is_dupack, rexmit);
2837
		tcp_identify_packet_loss(sk, ack_flag);
2838 2839
		if (!(icsk->icsk_ca_state == TCP_CA_Open ||
		      (*ack_flag & FLAG_LOST_RETRANS)))
L
Linus Torvalds 已提交
2840
			return;
2841
		/* Change state if cwnd is undone or retransmits are lost */
2842
		/* fall through */
L
Linus Torvalds 已提交
2843
	default:
2844
		if (tcp_is_reno(tp)) {
2845
			if (flag & FLAG_SND_UNA_ADVANCED)
L
Linus Torvalds 已提交
2846 2847
				tcp_reset_reno_sack(tp);
			if (is_dupack)
2848
				tcp_add_reno_sack(sk);
L
Linus Torvalds 已提交
2849 2850
		}

2851
		if (icsk->icsk_ca_state <= TCP_CA_Disorder)
2852
			tcp_try_undo_dsack(sk);
L
Linus Torvalds 已提交
2853

2854
		tcp_identify_packet_loss(sk, ack_flag);
2855
		if (!tcp_time_to_recover(sk, flag)) {
2856
			tcp_try_to_open(sk, flag);
L
Linus Torvalds 已提交
2857 2858 2859
			return;
		}

J
John Heffner 已提交
2860 2861 2862
		/* MTU probe failure: don't reduce cwnd */
		if (icsk->icsk_ca_state < TCP_CA_CWR &&
		    icsk->icsk_mtup.probe_size &&
2863
		    tp->snd_una == tp->mtu_probe.probe_seq_start) {
J
John Heffner 已提交
2864 2865 2866 2867 2868 2869 2870
			tcp_mtup_probe_failed(sk);
			/* Restores the reduction we did in tcp_mtup_probe() */
			tp->snd_cwnd++;
			tcp_simple_retransmit(sk);
			return;
		}

L
Linus Torvalds 已提交
2871
		/* Otherwise enter Recovery state */
2872
		tcp_enter_recovery(sk, (flag & FLAG_ECE));
2873
		fast_rexmit = 1;
L
Linus Torvalds 已提交
2874 2875
	}

2876
	if (!tcp_is_rack(sk) && do_lost)
2877
		tcp_update_scoreboard(sk, fast_rexmit);
2878
	*rexmit = REXMIT_LOST;
L
Linus Torvalds 已提交
2879 2880
}

2881
static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us, const int flag)
2882
{
2883
	u32 wlen = sock_net(sk)->ipv4.sysctl_tcp_min_rtt_wlen * HZ;
2884 2885
	struct tcp_sock *tp = tcp_sk(sk);

2886 2887 2888 2889 2890 2891 2892
	if ((flag & FLAG_ACK_MAYBE_DELAYED) && rtt_us > tcp_min_rtt(tp)) {
		/* If the remote keeps returning delayed ACKs, eventually
		 * the min filter would pick it up and overestimate the
		 * prop. delay when it expires. Skip suspected delayed ACKs.
		 */
		return;
	}
2893
	minmax_running_min(&tp->rtt_min, wlen, tcp_jiffies32,
2894
			   rtt_us ? : jiffies_to_usecs(1));
2895 2896
}

2897 2898 2899
static bool tcp_ack_update_rtt(struct sock *sk, const int flag,
			       long seq_rtt_us, long sack_rtt_us,
			       long ca_rtt_us, struct rate_sample *rs)
2900
{
2901 2902 2903 2904 2905 2906 2907
	const struct tcp_sock *tp = tcp_sk(sk);

	/* Prefer RTT measured from ACK's timing to TS-ECR. This is because
	 * broken middle-boxes or peers may corrupt TS-ECR fields. But
	 * Karn's algorithm forbids taking RTT if some retransmitted data
	 * is acked (RFC6298).
	 */
2908 2909
	if (seq_rtt_us < 0)
		seq_rtt_us = sack_rtt_us;
Y
Yuchung Cheng 已提交
2910

L
Linus Torvalds 已提交
2911 2912 2913 2914 2915 2916
	/* RTTM Rule: A TSecr value received in a segment is used to
	 * update the averaged RTT measurement only if the segment
	 * acknowledges some new data, i.e., only if it advances the
	 * left edge of the send window.
	 * See draft-ietf-tcplw-high-performance-00, section 3.3.
	 */
2917
	if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
2918 2919 2920 2921 2922 2923
	    flag & FLAG_ACKED) {
		u32 delta = tcp_time_stamp(tp) - tp->rx_opt.rcv_tsecr;
		u32 delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);

		seq_rtt_us = ca_rtt_us = delta_us;
	}
2924
	rs->rtt_us = ca_rtt_us; /* RTT of last (S)ACKed packet (or -1) */
2925
	if (seq_rtt_us < 0)
Y
Yuchung Cheng 已提交
2926
		return false;
L
Linus Torvalds 已提交
2927

2928 2929 2930 2931
	/* ca_rtt_us >= 0 is counting on the invariant that ca_rtt_us is
	 * always taken together with ACK, SACK, or TS-opts. Any negative
	 * values will be skipped with the seq_rtt_us < 0 check above.
	 */
2932
	tcp_update_rtt_min(sk, ca_rtt_us, flag);
2933
	tcp_rtt_estimator(sk, seq_rtt_us);
2934
	tcp_set_rto(sk);
L
Linus Torvalds 已提交
2935

2936 2937
	/* RFC6298: only reset backoff on valid RTT measurement. */
	inet_csk(sk)->icsk_backoff = 0;
Y
Yuchung Cheng 已提交
2938
	return true;
L
Linus Torvalds 已提交
2939 2940
}

2941
/* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */
Y
Yuchung Cheng 已提交
2942
void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req)
2943
{
2944
	struct rate_sample rs;
Y
Yuchung Cheng 已提交
2945
	long rtt_us = -1L;
2946

2947 2948
	if (req && !req->num_retrans && tcp_rsk(req)->snt_synack)
		rtt_us = tcp_stamp_us_delta(tcp_clock_us(), tcp_rsk(req)->snt_synack);
Y
Yuchung Cheng 已提交
2949

2950
	tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, rtt_us, -1L, rtt_us, &rs);
2951 2952
}

Y
Yuchung Cheng 已提交
2953

2954
static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
L
Linus Torvalds 已提交
2955
{
2956
	const struct inet_connection_sock *icsk = inet_csk(sk);
2957 2958

	icsk->icsk_ca_ops->cong_avoid(sk, ack, acked);
2959
	tcp_sk(sk)->snd_cwnd_stamp = tcp_jiffies32;
L
Linus Torvalds 已提交
2960 2961 2962 2963 2964
}

/* Restart timer after forward progress on connection.
 * RFC2988 recommends to restart timer to now+rto.
 */
2965
void tcp_rearm_rto(struct sock *sk)
L
Linus Torvalds 已提交
2966
{
N
Nandita Dukkipati 已提交
2967
	const struct inet_connection_sock *icsk = inet_csk(sk);
2968
	struct tcp_sock *tp = tcp_sk(sk);
2969

2970 2971 2972 2973 2974 2975
	/* If the retrans timer is currently being used by Fast Open
	 * for SYN-ACK retrans purpose, stay put.
	 */
	if (tp->fastopen_rsk)
		return;

L
Linus Torvalds 已提交
2976
	if (!tp->packets_out) {
2977
		inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS);
L
Linus Torvalds 已提交
2978
	} else {
2979 2980
		u32 rto = inet_csk(sk)->icsk_rto;
		/* Offset the time elapsed after installing regular RTO */
Y
Yuchung Cheng 已提交
2981
		if (icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
N
Nandita Dukkipati 已提交
2982
		    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2983
			s64 delta_us = tcp_rto_delta_us(sk);
E
Eric Dumazet 已提交
2984
			/* delta_us may not be positive if the socket is locked
N
Nandita Dukkipati 已提交
2985
			 * when the retrans timer fires and is rescheduled.
2986
			 */
2987
			rto = usecs_to_jiffies(max_t(int, delta_us, 1));
2988 2989 2990
		}
		inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
					  TCP_RTO_MAX);
L
Linus Torvalds 已提交
2991
	}
2992 2993
}

2994 2995 2996
/* Try to schedule a loss probe; if that doesn't work, then schedule an RTO. */
static void tcp_set_xmit_timer(struct sock *sk)
{
2997
	if (!tcp_schedule_loss_probe(sk, true))
2998 2999 3000
		tcp_rearm_rto(sk);
}

3001
/* If we get here, the whole TSO packet has not been acked. */
3002
static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
3003 3004
{
	struct tcp_sock *tp = tcp_sk(sk);
3005
	u32 packets_acked;
L
Linus Torvalds 已提交
3006

3007
	BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una));
L
Linus Torvalds 已提交
3008 3009

	packets_acked = tcp_skb_pcount(skb);
3010
	if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
L
Linus Torvalds 已提交
3011 3012 3013 3014 3015
		return 0;
	packets_acked -= tcp_skb_pcount(skb);

	if (packets_acked) {
		BUG_ON(tcp_skb_pcount(skb) == 0);
3016
		BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq));
L
Linus Torvalds 已提交
3017 3018
	}

3019
	return packets_acked;
L
Linus Torvalds 已提交
3020 3021
}

3022 3023 3024 3025 3026 3027
static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb,
			   u32 prior_snd_una)
{
	const struct skb_shared_info *shinfo;

	/* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */
3028
	if (likely(!TCP_SKB_CB(skb)->txstamp_ack))
3029 3030 3031
		return;

	shinfo = skb_shinfo(skb);
3032
	if (!before(shinfo->tskey, prior_snd_una) &&
3033 3034 3035 3036 3037
	    before(shinfo->tskey, tcp_sk(sk)->snd_una)) {
		tcp_skb_tsorted_save(skb) {
			__skb_tstamp_tx(skb, NULL, sk, SCM_TSTAMP_ACK);
		} tcp_skb_tsorted_restore(skb);
	}
3038 3039
}

3040 3041 3042 3043
/* Remove acknowledged frames from the retransmission queue. If our packet
 * is before the ack sequence we can discard it as it's confirmed to have
 * arrived at the other end.
 */
3044 3045
static int tcp_clean_rtx_queue(struct sock *sk, u32 prior_fack,
			       u32 prior_snd_una,
3046
			       struct tcp_sacktag_state *sack)
L
Linus Torvalds 已提交
3047
{
3048
	const struct inet_connection_sock *icsk = inet_csk(sk);
3049
	u64 first_ackt, last_ackt;
3050 3051
	struct tcp_sock *tp = tcp_sk(sk);
	u32 prior_sacked = tp->sacked_out;
3052
	u32 reord = tp->snd_nxt; /* lowest acked un-retx un-sacked seq */
3053
	struct sk_buff *skb, *next;
3054
	bool fully_acked = true;
3055
	long sack_rtt_us = -1L;
3056
	long seq_rtt_us = -1L;
3057
	long ca_rtt_us = -1L;
3058
	u32 pkts_acked = 0;
3059
	u32 last_in_flight = 0;
3060
	bool rtt_update;
3061 3062
	int flag = 0;

3063
	first_ackt = 0;
L
Linus Torvalds 已提交
3064

3065
	for (skb = skb_rb_first(&sk->tcp_rtx_queue); skb; skb = next) {
3066
		struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
3067
		const u32 start_seq = scb->seq;
3068
		u8 sacked = scb->sacked;
3069
		u32 acked_pcount;
L
Linus Torvalds 已提交
3070

3071
		tcp_ack_tstamp(sk, skb, prior_snd_una);
3072

3073
		/* Determine how many packets and what bytes were acked, tso and else */
L
Linus Torvalds 已提交
3074
		if (after(scb->end_seq, tp->snd_una)) {
3075 3076 3077 3078
			if (tcp_skb_pcount(skb) == 1 ||
			    !after(tp->snd_una, scb->seq))
				break;

3079 3080
			acked_pcount = tcp_tso_acked(sk, skb);
			if (!acked_pcount)
3081
				break;
E
Eric Dumazet 已提交
3082
			fully_acked = false;
3083
		} else {
3084
			acked_pcount = tcp_skb_pcount(skb);
L
Linus Torvalds 已提交
3085 3086
		}

3087
		if (unlikely(sacked & TCPCB_RETRANS)) {
3088
			if (sacked & TCPCB_SACKED_RETRANS)
3089
				tp->retrans_out -= acked_pcount;
3090
			flag |= FLAG_RETRANS_DATA_ACKED;
3091
		} else if (!(sacked & TCPCB_SACKED_ACKED)) {
3092
			last_ackt = skb->skb_mstamp;
3093 3094
			WARN_ON_ONCE(last_ackt == 0);
			if (!first_ackt)
3095 3096
				first_ackt = last_ackt;

3097
			last_in_flight = TCP_SKB_CB(skb)->tx.in_flight;
3098 3099
			if (before(start_seq, reord))
				reord = start_seq;
3100 3101
			if (!after(scb->end_seq, tp->high_seq))
				flag |= FLAG_ORIG_SACK_ACKED;
3102
		}
3103

Y
Yuchung Cheng 已提交
3104
		if (sacked & TCPCB_SACKED_ACKED) {
3105
			tp->sacked_out -= acked_pcount;
Y
Yuchung Cheng 已提交
3106 3107 3108
		} else if (tcp_is_sack(tp)) {
			tp->delivered += acked_pcount;
			if (!tcp_skb_spurious_retrans(tp, skb))
3109
				tcp_rack_advance(tp, sacked, scb->end_seq,
3110
						 skb->skb_mstamp);
Y
Yuchung Cheng 已提交
3111
		}
3112
		if (sacked & TCPCB_LOST)
3113
			tp->lost_out -= acked_pcount;
3114

3115 3116
		tp->packets_out -= acked_pcount;
		pkts_acked += acked_pcount;
3117
		tcp_rate_skb_delivered(sk, skb, sack->rate);
3118

3119 3120 3121 3122 3123 3124 3125
		/* Initial outgoing SYN's get put onto the write_queue
		 * just like anything else we transmit.  It is not
		 * true data, and if we misinform our callers that
		 * this ACK acks real data, we will erroneously exit
		 * connection startup slow start one packet too
		 * quickly.  This is severely frowned upon behavior.
		 */
3126
		if (likely(!(scb->tcp_flags & TCPHDR_SYN))) {
3127 3128 3129 3130 3131 3132
			flag |= FLAG_DATA_ACKED;
		} else {
			flag |= FLAG_SYN_ACKED;
			tp->retrans_stamp = 0;
		}

3133 3134 3135
		if (!fully_acked)
			break;

3136
		next = skb_rb_next(skb);
3137
		if (unlikely(skb == tp->retransmit_skb_hint))
3138
			tp->retransmit_skb_hint = NULL;
3139
		if (unlikely(skb == tp->lost_skb_hint))
3140
			tp->lost_skb_hint = NULL;
3141
		tcp_rtx_queue_unlink_and_free(skb, sk);
L
Linus Torvalds 已提交
3142 3143
	}

3144 3145 3146
	if (!skb)
		tcp_chrono_stop(sk, TCP_CHRONO_BUSY);

I
Ilpo Järvinen 已提交
3147 3148 3149
	if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una)))
		tp->snd_up = tp->snd_una;

3150 3151 3152
	if (skb && (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
		flag |= FLAG_SACK_RENEGING;

3153 3154 3155
	if (likely(first_ackt) && !(flag & FLAG_RETRANS_DATA_ACKED)) {
		seq_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, first_ackt);
		ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, last_ackt);
3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166

		if (pkts_acked == 1 && last_in_flight < tp->mss_cache &&
		    last_in_flight && !prior_sacked && fully_acked &&
		    sack->rate->prior_delivered + 1 == tp->delivered &&
		    !(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) {
			/* Conservatively mark a delayed ACK. It's typically
			 * from a lone runt packet over the round trip to
			 * a receiver w/o out-of-order or CE events.
			 */
			flag |= FLAG_ACK_MAYBE_DELAYED;
		}
3167
	}
3168 3169 3170
	if (sack->first_sackt) {
		sack_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->first_sackt);
		ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->last_sackt);
3171
	}
3172
	rtt_update = tcp_ack_update_rtt(sk, flag, seq_rtt_us, sack_rtt_us,
3173
					ca_rtt_us, sack->rate);
Y
Yuchung Cheng 已提交
3174

3175
	if (flag & FLAG_ACKED) {
3176
		flag |= FLAG_SET_XMIT_TIMER;  /* set TLP or RTO timer */
3177 3178 3179 3180 3181
		if (unlikely(icsk->icsk_mtup.probe_size &&
			     !after(tp->mtu_probe.probe_seq_end, tp->snd_una))) {
			tcp_mtup_probe_success(sk);
		}

3182 3183 3184
		if (tcp_is_reno(tp)) {
			tcp_remove_reno_sacks(sk, pkts_acked);
		} else {
3185 3186
			int delta;

3187
			/* Non-retransmitted hole got filled? That's reordering */
3188 3189
			if (before(reord, prior_fack))
				tcp_check_sack_reordering(sk, reord, 0);
3190

Y
Yuchung Cheng 已提交
3191
			delta = prior_sacked - tp->sacked_out;
3192
			tp->lost_cnt_hint -= min(tp->lost_cnt_hint, delta);
3193
		}
3194
	} else if (skb && rtt_update && sack_rtt_us >= 0 &&
3195
		   sack_rtt_us > tcp_stamp_us_delta(tp->tcp_mstamp, skb->skb_mstamp)) {
3196 3197 3198 3199
		/* Do not re-arm RTO if the sack RTT is measured from data sent
		 * after when the head was last (re)transmitted. Otherwise the
		 * timeout may continue to extend in loss recovery.
		 */
3200
		flag |= FLAG_SET_XMIT_TIMER;  /* set TLP or RTO timer */
L
Linus Torvalds 已提交
3201 3202
	}

3203 3204
	if (icsk->icsk_ca_ops->pkts_acked) {
		struct ack_sample sample = { .pkts_acked = pkts_acked,
3205
					     .rtt_us = sack->rate->rtt_us,
3206
					     .in_flight = last_in_flight };
3207 3208 3209

		icsk->icsk_ca_ops->pkts_acked(sk, &sample);
	}
3210

L
Linus Torvalds 已提交
3211
#if FASTRETRANS_DEBUG > 0
3212 3213 3214
	WARN_ON((int)tp->sacked_out < 0);
	WARN_ON((int)tp->lost_out < 0);
	WARN_ON((int)tp->retrans_out < 0);
3215
	if (!tp->packets_out && tcp_is_sack(tp)) {
S
Stephen Hemminger 已提交
3216
		icsk = inet_csk(sk);
L
Linus Torvalds 已提交
3217
		if (tp->lost_out) {
3218 3219
			pr_debug("Leak l=%u %d\n",
				 tp->lost_out, icsk->icsk_ca_state);
L
Linus Torvalds 已提交
3220 3221 3222
			tp->lost_out = 0;
		}
		if (tp->sacked_out) {
3223 3224
			pr_debug("Leak s=%u %d\n",
				 tp->sacked_out, icsk->icsk_ca_state);
L
Linus Torvalds 已提交
3225 3226 3227
			tp->sacked_out = 0;
		}
		if (tp->retrans_out) {
3228 3229
			pr_debug("Leak r=%u %d\n",
				 tp->retrans_out, icsk->icsk_ca_state);
L
Linus Torvalds 已提交
3230 3231 3232 3233
			tp->retrans_out = 0;
		}
	}
#endif
3234
	return flag;
L
Linus Torvalds 已提交
3235 3236 3237 3238
}

static void tcp_ack_probe(struct sock *sk)
{
3239
	struct inet_connection_sock *icsk = inet_csk(sk);
3240 3241
	struct sk_buff *head = tcp_send_head(sk);
	const struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
3242 3243

	/* Was it a usable window open? */
3244 3245 3246
	if (!head)
		return;
	if (!after(TCP_SKB_CB(head)->end_seq, tcp_wnd_end(tp))) {
3247 3248
		icsk->icsk_backoff = 0;
		inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0);
L
Linus Torvalds 已提交
3249 3250 3251 3252
		/* Socket must be waked up by subsequent tcp_data_snd_check().
		 * This function is not for random using!
		 */
	} else {
3253
		unsigned long when = tcp_probe0_when(sk, TCP_RTO_MAX);
3254

3255
		inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3256
					  when, TCP_RTO_MAX);
L
Linus Torvalds 已提交
3257 3258 3259
	}
}

3260
static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag)
L
Linus Torvalds 已提交
3261
{
E
Eric Dumazet 已提交
3262 3263
	return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) ||
		inet_csk(sk)->icsk_ca_state != TCP_CA_Open;
L
Linus Torvalds 已提交
3264 3265
}

3266
/* Decide wheather to run the increase function of congestion control. */
3267
static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
L
Linus Torvalds 已提交
3268
{
3269 3270
	/* If reordering is high then always grow cwnd whenever data is
	 * delivered regardless of its ordering. Otherwise stay conservative
3271
	 * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/
3272 3273 3274
	 * new SACK or ECE mark may first advance cwnd here and later reduce
	 * cwnd in tcp_fastretrans_alert() based on more states.
	 */
3275
	if (tcp_sk(sk)->reordering > sock_net(sk)->ipv4.sysctl_tcp_reordering)
3276 3277
		return flag & FLAG_FORWARD_PROGRESS;

3278
	return flag & FLAG_DATA_ACKED;
L
Linus Torvalds 已提交
3279 3280
}

Y
Yuchung Cheng 已提交
3281 3282 3283 3284 3285 3286
/* The "ultimate" congestion control function that aims to replace the rigid
 * cwnd increase and decrease control (tcp_cong_avoid,tcp_*cwnd_reduction).
 * It's called toward the end of processing an ACK with precise rate
 * information. All transmission or retransmission are delayed afterwards.
 */
static void tcp_cong_control(struct sock *sk, u32 ack, u32 acked_sacked,
3287
			     int flag, const struct rate_sample *rs)
Y
Yuchung Cheng 已提交
3288
{
3289 3290 3291 3292 3293 3294 3295
	const struct inet_connection_sock *icsk = inet_csk(sk);

	if (icsk->icsk_ca_ops->cong_control) {
		icsk->icsk_ca_ops->cong_control(sk, rs);
		return;
	}

Y
Yuchung Cheng 已提交
3296 3297 3298 3299 3300 3301 3302 3303 3304 3305
	if (tcp_in_cwnd_reduction(sk)) {
		/* Reduce cwnd if state mandates */
		tcp_cwnd_reduction(sk, acked_sacked, flag);
	} else if (tcp_may_raise_cwnd(sk, flag)) {
		/* Advance cwnd if state allows */
		tcp_cong_avoid(sk, ack, acked_sacked);
	}
	tcp_update_pacing_rate(sk);
}

L
Linus Torvalds 已提交
3306 3307 3308
/* Check that window update is acceptable.
 * The function assumes that snd_una<=ack<=snd_next.
 */
3309
static inline bool tcp_may_update_window(const struct tcp_sock *tp,
3310 3311
					const u32 ack, const u32 ack_seq,
					const u32 nwin)
L
Linus Torvalds 已提交
3312
{
E
Eric Dumazet 已提交
3313
	return	after(ack, tp->snd_una) ||
L
Linus Torvalds 已提交
3314
		after(ack_seq, tp->snd_wl1) ||
E
Eric Dumazet 已提交
3315
		(ack_seq == tp->snd_wl1 && nwin > tp->snd_wnd);
L
Linus Torvalds 已提交
3316 3317
}

3318 3319 3320 3321 3322
/* If we update tp->snd_una, also update tp->bytes_acked */
static void tcp_snd_una_update(struct tcp_sock *tp, u32 ack)
{
	u32 delta = ack - tp->snd_una;

3323
	sock_owned_by_me((struct sock *)tp);
3324 3325 3326 3327
	tp->bytes_acked += delta;
	tp->snd_una = ack;
}

3328 3329 3330 3331 3332
/* If we update tp->rcv_nxt, also update tp->bytes_received */
static void tcp_rcv_nxt_update(struct tcp_sock *tp, u32 seq)
{
	u32 delta = seq - tp->rcv_nxt;

3333
	sock_owned_by_me((struct sock *)tp);
3334 3335 3336 3337
	tp->bytes_received += delta;
	tp->rcv_nxt = seq;
}

L
Linus Torvalds 已提交
3338 3339 3340 3341 3342
/* Update our send window.
 *
 * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2
 * and in FreeBSD. NetBSD's one is even worse.) is wrong.
 */
3343
static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack,
3344
				 u32 ack_seq)
L
Linus Torvalds 已提交
3345
{
3346
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
3347
	int flag = 0;
3348
	u32 nwin = ntohs(tcp_hdr(skb)->window);
L
Linus Torvalds 已提交
3349

3350
	if (likely(!tcp_hdr(skb)->syn))
L
Linus Torvalds 已提交
3351 3352 3353 3354
		nwin <<= tp->rx_opt.snd_wscale;

	if (tcp_may_update_window(tp, ack, ack_seq, nwin)) {
		flag |= FLAG_WIN_UPDATE;
3355
		tcp_update_wl(tp, ack_seq);
L
Linus Torvalds 已提交
3356 3357 3358 3359

		if (tp->snd_wnd != nwin) {
			tp->snd_wnd = nwin;

3360 3361 3362 3363 3364 3365
			/* Note, it is the only place, where
			 * fast path is recovered for sending TCP.
			 */
			tp->pred_flags = 0;
			tcp_fast_path_check(sk);

3366
			if (!tcp_write_queue_empty(sk))
3367 3368
				tcp_slow_start_after_idle_check(sk);

L
Linus Torvalds 已提交
3369 3370
			if (nwin > tp->max_window) {
				tp->max_window = nwin;
3371
				tcp_sync_mss(sk, inet_csk(sk)->icsk_pmtu_cookie);
L
Linus Torvalds 已提交
3372 3373 3374 3375
			}
		}
	}

3376
	tcp_snd_una_update(tp, ack);
L
Linus Torvalds 已提交
3377 3378 3379 3380

	return flag;
}

3381 3382 3383 3384
static bool __tcp_oow_rate_limited(struct net *net, int mib_idx,
				   u32 *last_oow_ack_time)
{
	if (*last_oow_ack_time) {
3385
		s32 elapsed = (s32)(tcp_jiffies32 - *last_oow_ack_time);
3386

3387
		if (0 <= elapsed && elapsed < net->ipv4.sysctl_tcp_invalid_ratelimit) {
3388 3389 3390 3391 3392
			NET_INC_STATS(net, mib_idx);
			return true;	/* rate-limited: don't send yet! */
		}
	}

3393
	*last_oow_ack_time = tcp_jiffies32;
3394 3395 3396 3397

	return false;	/* not rate-limited: go ahead, send dupack now! */
}

3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410
/* Return true if we're currently rate-limiting out-of-window ACKs and
 * thus shouldn't send a dupack right now. We rate-limit dupacks in
 * response to out-of-window SYNs or ACKs to mitigate ACK loops or DoS
 * attacks that send repeated SYNs or ACKs for the same connection. To
 * do this, we do not send a duplicate SYNACK or ACK if the remote
 * endpoint is sending out-of-window SYNs or pure ACKs at a high rate.
 */
bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
			  int mib_idx, u32 *last_oow_ack_time)
{
	/* Data packets without SYNs are not likely part of an ACK loop. */
	if ((TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq) &&
	    !tcp_hdr(skb)->syn)
3411
		return false;
3412

3413
	return __tcp_oow_rate_limited(net, mib_idx, last_oow_ack_time);
3414 3415
}

3416
/* RFC 5961 7 [ACK Throttling] */
3417
static void tcp_send_challenge_ack(struct sock *sk, const struct sk_buff *skb)
3418 3419 3420 3421
{
	/* unprotected vars, we dont care of overwrites */
	static u32 challenge_timestamp;
	static unsigned int challenge_count;
3422
	struct tcp_sock *tp = tcp_sk(sk);
3423
	struct net *net = sock_net(sk);
3424
	u32 count, now;
3425 3426

	/* First check our per-socket dupack rate limit. */
3427
	if (__tcp_oow_rate_limited(net,
3428 3429
				   LINUX_MIB_TCPACKSKIPPEDCHALLENGE,
				   &tp->last_oow_ack_time))
3430
		return;
3431

3432
	/* Then check host-wide RFC 5961 rate limit. */
3433
	now = jiffies / HZ;
3434
	if (now != challenge_timestamp) {
3435 3436
		u32 ack_limit = net->ipv4.sysctl_tcp_challenge_ack_limit;
		u32 half = (ack_limit + 1) >> 1;
3437

3438
		challenge_timestamp = now;
3439
		WRITE_ONCE(challenge_count, half + prandom_u32_max(ack_limit));
3440
	}
3441 3442 3443
	count = READ_ONCE(challenge_count);
	if (count > 0) {
		WRITE_ONCE(challenge_count, count - 1);
3444
		NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK);
3445 3446 3447 3448
		tcp_send_ack(sk);
	}
}

3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469
static void tcp_store_ts_recent(struct tcp_sock *tp)
{
	tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval;
	tp->rx_opt.ts_recent_stamp = get_seconds();
}

static void tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq)
{
	if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) {
		/* PAWS bug workaround wrt. ACK frames, the PAWS discard
		 * extra check below makes sure this can only happen
		 * for pure ACK frames.  -DaveM
		 *
		 * Not only, also it occurs for expired timestamps.
		 */

		if (tcp_paws_check(&tp->rx_opt, 0))
			tcp_store_ts_recent(tp);
	}
}

N
Nandita Dukkipati 已提交
3470
/* This routine deals with acks during a TLP episode.
3471 3472
 * We mark the end of a TLP episode on receiving TLP dupack or when
 * ack is after tlp_high_seq.
N
Nandita Dukkipati 已提交
3473 3474 3475 3476 3477 3478
 * Ref: loss detection algorithm in draft-dukkipati-tcpm-tcp-loss-probe.
 */
static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag)
{
	struct tcp_sock *tp = tcp_sk(sk);

3479
	if (before(ack, tp->tlp_high_seq))
N
Nandita Dukkipati 已提交
3480 3481
		return;

3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492
	if (flag & FLAG_DSACKING_ACK) {
		/* This DSACK means original and TLP probe arrived; no loss */
		tp->tlp_high_seq = 0;
	} else if (after(ack, tp->tlp_high_seq)) {
		/* ACK advances: there was a loss, so reduce cwnd. Reset
		 * tlp_high_seq in tcp_init_cwnd_reduction()
		 */
		tcp_init_cwnd_reduction(sk);
		tcp_set_ca_state(sk, TCP_CA_CWR);
		tcp_end_cwnd_reduction(sk);
		tcp_try_keep_open(sk);
3493
		NET_INC_STATS(sock_net(sk),
3494
				LINUX_MIB_TCPLOSSPROBERECOVERY);
3495 3496 3497
	} else if (!(flag & (FLAG_SND_UNA_ADVANCED |
			     FLAG_NOT_DUP | FLAG_DATA_SACKED))) {
		/* Pure dupack: original and TLP probe arrived; no loss */
N
Nandita Dukkipati 已提交
3498 3499 3500 3501
		tp->tlp_high_seq = 0;
	}
}

3502 3503 3504 3505 3506 3507 3508 3509
static inline void tcp_in_ack_event(struct sock *sk, u32 flags)
{
	const struct inet_connection_sock *icsk = inet_csk(sk);

	if (icsk->icsk_ca_ops->in_ack_event)
		icsk->icsk_ca_ops->in_ack_event(sk, flags);
}

3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530
/* Congestion control has updated the cwnd already. So if we're in
 * loss recovery then now we do any new sends (for FRTO) or
 * retransmits (for CA_Loss or CA_recovery) that make sense.
 */
static void tcp_xmit_recovery(struct sock *sk, int rexmit)
{
	struct tcp_sock *tp = tcp_sk(sk);

	if (rexmit == REXMIT_NONE)
		return;

	if (unlikely(rexmit == 2)) {
		__tcp_push_pending_frames(sk, tcp_current_mss(sk),
					  TCP_NAGLE_OFF);
		if (after(tp->snd_nxt, tp->high_seq))
			return;
		tp->frto = 0;
	}
	tcp_xmit_retransmit_queue(sk);
}

3531 3532 3533
/* Returns the number of packets newly acked or sacked by the current ACK */
static u32 tcp_newly_delivered(struct sock *sk, u32 prior_delivered, int flag)
{
3534
	const struct net *net = sock_net(sk);
3535 3536 3537 3538
	struct tcp_sock *tp = tcp_sk(sk);
	u32 delivered;

	delivered = tp->delivered - prior_delivered;
3539 3540
	NET_ADD_STATS(net, LINUX_MIB_TCPDELIVERED, delivered);
	if (flag & FLAG_ECE) {
3541
		tp->delivered_ce += delivered;
3542 3543
		NET_ADD_STATS(net, LINUX_MIB_TCPDELIVEREDCE, delivered);
	}
3544 3545 3546
	return delivered;
}

L
Linus Torvalds 已提交
3547
/* This routine deals with incoming acks, but not outgoing ones. */
3548
static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
L
Linus Torvalds 已提交
3549
{
3550
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
3551
	struct tcp_sock *tp = tcp_sk(sk);
3552
	struct tcp_sacktag_state sack_state;
3553
	struct rate_sample rs = { .prior_delivered = 0 };
L
Linus Torvalds 已提交
3554
	u32 prior_snd_una = tp->snd_una;
3555
	bool is_sack_reneg = tp->is_sack_reneg;
L
Linus Torvalds 已提交
3556 3557
	u32 ack_seq = TCP_SKB_CB(skb)->seq;
	u32 ack = TCP_SKB_CB(skb)->ack_seq;
3558
	bool is_dupack = false;
3559
	int prior_packets = tp->packets_out;
3560 3561
	u32 delivered = tp->delivered;
	u32 lost = tp->lost;
3562
	int rexmit = REXMIT_NONE; /* Flag to (re)transmit to recover losses */
3563
	u32 prior_fack;
3564

3565
	sack_state.first_sackt = 0;
3566
	sack_state.rate = &rs;
L
Linus Torvalds 已提交
3567

3568 3569
	/* We very likely will need to access rtx queue. */
	prefetch(sk->tcp_rtx_queue.rb_node);
3570

3571
	/* If the ack is older than previous acks
L
Linus Torvalds 已提交
3572 3573
	 * then we can probably ignore it.
	 */
3574 3575 3576
	if (before(ack, prior_snd_una)) {
		/* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */
		if (before(ack, prior_snd_una - tp->max_window)) {
3577 3578
			if (!(flag & FLAG_NO_CHALLENGE_ACK))
				tcp_send_challenge_ack(sk, skb);
3579 3580
			return -1;
		}
L
Linus Torvalds 已提交
3581
		goto old_ack;
3582
	}
L
Linus Torvalds 已提交
3583

3584 3585 3586 3587 3588 3589
	/* If the ack includes data we haven't sent yet, discard
	 * this segment (RFC793 Section 3.9).
	 */
	if (after(ack, tp->snd_nxt))
		goto invalid_ack;

3590
	if (after(ack, prior_snd_una)) {
3591
		flag |= FLAG_SND_UNA_ADVANCED;
3592
		icsk->icsk_retransmits = 0;
I
Ilya Lesokhin 已提交
3593 3594 3595 3596 3597 3598

#if IS_ENABLED(CONFIG_TLS_DEVICE)
		if (static_branch_unlikely(&clean_acked_data_enabled))
			if (icsk->icsk_clean_acked)
				icsk->icsk_clean_acked(sk, ack);
#endif
3599
	}
3600

E
Eric Dumazet 已提交
3601
	prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una;
3602
	rs.prior_in_flight = tcp_packets_in_flight(tp);
3603

3604 3605 3606 3607 3608 3609
	/* ts_recent update must be made after we are sure that the packet
	 * is in window.
	 */
	if (flag & FLAG_UPDATE_TS_RECENT)
		tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);

3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622
	if (!(flag & FLAG_SLOWPATH) && after(ack, prior_snd_una)) {
		/* Window is constant, pure forward advance.
		 * No more checks are required.
		 * Note, we use the fact that SND.UNA>=SND.WL2.
		 */
		tcp_update_wl(tp, ack_seq);
		tcp_snd_una_update(tp, ack);
		flag |= FLAG_WIN_UPDATE;

		tcp_in_ack_event(sk, CA_ACK_WIN_UPDATE);

		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPACKS);
	} else {
3623
		u32 ack_ev_flags = CA_ACK_SLOWPATH;
L
Linus Torvalds 已提交
3624

3625 3626 3627 3628
		if (ack_seq != TCP_SKB_CB(skb)->end_seq)
			flag |= FLAG_DATA;
		else
			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPUREACKS);
L
Linus Torvalds 已提交
3629

3630
		flag |= tcp_ack_update_window(sk, skb, ack, ack_seq);
L
Linus Torvalds 已提交
3631

3632 3633 3634
		if (TCP_SKB_CB(skb)->sacked)
			flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
							&sack_state);
3635

3636 3637 3638 3639 3640 3641 3642
		if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb))) {
			flag |= FLAG_ECE;
			ack_ev_flags |= CA_ACK_ECE;
		}

		if (flag & FLAG_WIN_UPDATE)
			ack_ev_flags |= CA_ACK_WIN_UPDATE;
L
Linus Torvalds 已提交
3643

3644 3645
		tcp_in_ack_event(sk, ack_ev_flags);
	}
L
Linus Torvalds 已提交
3646 3647 3648 3649 3650

	/* We passed data and got it acked, remove any soft error
	 * log. Something worked...
	 */
	sk->sk_err_soft = 0;
3651
	icsk->icsk_probes_out = 0;
3652
	tp->rcv_tstamp = tcp_jiffies32;
L
Linus Torvalds 已提交
3653 3654 3655 3656
	if (!prior_packets)
		goto no_queue;

	/* See if we can take anything off of the retransmit queue. */
3657
	flag |= tcp_clean_rtx_queue(sk, prior_fack, prior_snd_una, &sack_state);
3658

3659
	tcp_rack_update_reo_wnd(sk, &rs);
3660

3661 3662 3663 3664 3665 3666
	if (tp->tlp_high_seq)
		tcp_process_tlp_ack(sk, ack, flag);
	/* If needed, reset TLP/RTO timer; RACK may later override this. */
	if (flag & FLAG_SET_XMIT_TIMER)
		tcp_set_xmit_timer(sk);

3667
	if (tcp_ack_is_dubious(sk, flag)) {
3668
		is_dupack = !(flag & (FLAG_SND_UNA_ADVANCED | FLAG_NOT_DUP));
3669 3670
		tcp_fastretrans_alert(sk, prior_snd_una, is_dupack, &flag,
				      &rexmit);
L
Linus Torvalds 已提交
3671
	}
N
Nandita Dukkipati 已提交
3672

3673 3674
	if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP))
		sk_dst_confirm(sk);
N
Nandita Dukkipati 已提交
3675

3676
	delivered = tcp_newly_delivered(sk, delivered, flag);
3677
	lost = tp->lost - lost;			/* freshly marked lost */
3678
	rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED);
3679
	tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate);
3680
	tcp_cong_control(sk, ack, delivered, flag, sack_state.rate);
3681
	tcp_xmit_recovery(sk, rexmit);
L
Linus Torvalds 已提交
3682 3683 3684
	return 1;

no_queue:
3685
	/* If data was DSACKed, see if we can undo a cwnd reduction. */
3686
	if (flag & FLAG_DSACKING_ACK) {
3687 3688
		tcp_fastretrans_alert(sk, prior_snd_una, is_dupack, &flag,
				      &rexmit);
3689 3690
		tcp_newly_delivered(sk, delivered, flag);
	}
L
Linus Torvalds 已提交
3691 3692 3693 3694
	/* If this ack opens up a zero window, clear backoff.  It was
	 * being used to time the probes, and is probably far higher than
	 * it needs to be for normal retransmission.
	 */
3695
	tcp_ack_probe(sk);
N
Nandita Dukkipati 已提交
3696 3697 3698

	if (tp->tlp_high_seq)
		tcp_process_tlp_ack(sk, ack, flag);
L
Linus Torvalds 已提交
3699 3700
	return 1;

3701 3702 3703 3704
invalid_ack:
	SOCK_DEBUG(sk, "Ack %u after %u:%u\n", ack, tp->snd_una, tp->snd_nxt);
	return -1;

L
Linus Torvalds 已提交
3705
old_ack:
3706 3707 3708
	/* If data was SACKed, tag it and see if we should send more data.
	 * If data was DSACKed, see if we can undo a cwnd reduction.
	 */
3709
	if (TCP_SKB_CB(skb)->sacked) {
Y
Yuchung Cheng 已提交
3710
		flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
3711
						&sack_state);
3712 3713
		tcp_fastretrans_alert(sk, prior_snd_una, is_dupack, &flag,
				      &rexmit);
3714
		tcp_newly_delivered(sk, delivered, flag);
3715
		tcp_xmit_recovery(sk, rexmit);
3716
	}
L
Linus Torvalds 已提交
3717

3718
	SOCK_DEBUG(sk, "Ack %u before %u:%u\n", ack, tp->snd_una, tp->snd_nxt);
L
Linus Torvalds 已提交
3719 3720 3721
	return 0;
}

3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738
static void tcp_parse_fastopen_option(int len, const unsigned char *cookie,
				      bool syn, struct tcp_fastopen_cookie *foc,
				      bool exp_opt)
{
	/* Valid only in SYN or SYN-ACK with an even length.  */
	if (!foc || !syn || len < 0 || (len & 1))
		return;

	if (len >= TCP_FASTOPEN_COOKIE_MIN &&
	    len <= TCP_FASTOPEN_COOKIE_MAX)
		memcpy(foc->val, cookie, len);
	else if (len != 0)
		len = -1;
	foc->len = len;
	foc->exp = exp_opt;
}

3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753
static void smc_parse_options(const struct tcphdr *th,
			      struct tcp_options_received *opt_rx,
			      const unsigned char *ptr,
			      int opsize)
{
#if IS_ENABLED(CONFIG_SMC)
	if (static_branch_unlikely(&tcp_have_smc)) {
		if (th->syn && !(opsize & 1) &&
		    opsize >= TCPOLEN_EXP_SMC_BASE &&
		    get_unaligned_be32(ptr) == TCPOPT_SMC_MAGIC)
			opt_rx->smc_ok = 1;
	}
#endif
}

L
Linus Torvalds 已提交
3754 3755 3756 3757
/* Look for tcp options. Normally only called on SYN and SYNACK packets.
 * But, this can also be called on packets in the established flow when
 * the fast version below fails.
 */
3758 3759
void tcp_parse_options(const struct net *net,
		       const struct sk_buff *skb,
C
Christoph Paasch 已提交
3760
		       struct tcp_options_received *opt_rx, int estab,
Y
Yuchung Cheng 已提交
3761
		       struct tcp_fastopen_cookie *foc)
L
Linus Torvalds 已提交
3762
{
3763 3764
	const unsigned char *ptr;
	const struct tcphdr *th = tcp_hdr(skb);
3765
	int length = (th->doff * 4) - sizeof(struct tcphdr);
L
Linus Torvalds 已提交
3766

3767
	ptr = (const unsigned char *)(th + 1);
L
Linus Torvalds 已提交
3768 3769
	opt_rx->saw_tstamp = 0;

S
Stephen Hemminger 已提交
3770
	while (length > 0) {
3771
		int opcode = *ptr++;
L
Linus Torvalds 已提交
3772 3773 3774
		int opsize;

		switch (opcode) {
3775 3776 3777 3778 3779 3780 3781 3782
		case TCPOPT_EOL:
			return;
		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
			length--;
			continue;
		default:
			opsize = *ptr++;
			if (opsize < 2) /* "silly options" */
L
Linus Torvalds 已提交
3783
				return;
3784 3785 3786 3787 3788
			if (opsize > length)
				return;	/* don't parse partial options */
			switch (opcode) {
			case TCPOPT_MSS:
				if (opsize == TCPOLEN_MSS && th->syn && !estab) {
3789
					u16 in_mss = get_unaligned_be16(ptr);
3790 3791 3792 3793 3794
					if (in_mss) {
						if (opt_rx->user_mss &&
						    opt_rx->user_mss < in_mss)
							in_mss = opt_rx->user_mss;
						opt_rx->mss_clamp = in_mss;
L
Linus Torvalds 已提交
3795
					}
3796 3797 3798 3799
				}
				break;
			case TCPOPT_WINDOW:
				if (opsize == TCPOLEN_WINDOW && th->syn &&
3800
				    !estab && net->ipv4.sysctl_tcp_window_scaling) {
3801 3802
					__u8 snd_wscale = *(__u8 *)ptr;
					opt_rx->wscale_ok = 1;
3803 3804
					if (snd_wscale > TCP_MAX_WSCALE) {
						net_info_ratelimited("%s: Illegal window scaling value %d > %u received\n",
3805
								     __func__,
3806 3807 3808
								     snd_wscale,
								     TCP_MAX_WSCALE);
						snd_wscale = TCP_MAX_WSCALE;
L
Linus Torvalds 已提交
3809
					}
3810 3811 3812 3813 3814 3815
					opt_rx->snd_wscale = snd_wscale;
				}
				break;
			case TCPOPT_TIMESTAMP:
				if ((opsize == TCPOLEN_TIMESTAMP) &&
				    ((estab && opt_rx->tstamp_ok) ||
3816
				     (!estab && net->ipv4.sysctl_tcp_timestamps))) {
3817
					opt_rx->saw_tstamp = 1;
3818 3819
					opt_rx->rcv_tsval = get_unaligned_be32(ptr);
					opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4);
3820 3821 3822 3823
				}
				break;
			case TCPOPT_SACK_PERM:
				if (opsize == TCPOLEN_SACK_PERM && th->syn &&
E
Eric Dumazet 已提交
3824
				    !estab && net->ipv4.sysctl_tcp_sack) {
3825
					opt_rx->sack_ok = TCP_SACK_SEEN;
3826 3827 3828
					tcp_sack_reset(opt_rx);
				}
				break;
L
Linus Torvalds 已提交
3829

3830 3831 3832 3833 3834 3835 3836
			case TCPOPT_SACK:
				if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) &&
				   !((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) &&
				   opt_rx->sack_ok) {
					TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th;
				}
				break;
3837
#ifdef CONFIG_TCP_MD5SIG
3838 3839 3840 3841 3842 3843
			case TCPOPT_MD5SIG:
				/*
				 * The MD5 Hash has already been
				 * checked (see tcp_v{4,6}_do_rcv()).
				 */
				break;
3844
#endif
3845 3846 3847 3848 3849 3850
			case TCPOPT_FASTOPEN:
				tcp_parse_fastopen_option(
					opsize - TCPOLEN_FASTOPEN_BASE,
					ptr, th->syn, foc, false);
				break;

Y
Yuchung Cheng 已提交
3851 3852
			case TCPOPT_EXP:
				/* Fast Open option shares code 254 using a
3853
				 * 16 bits magic number.
Y
Yuchung Cheng 已提交
3854
				 */
3855 3856 3857 3858 3859 3860
				if (opsize >= TCPOLEN_EXP_FASTOPEN_BASE &&
				    get_unaligned_be16(ptr) ==
				    TCPOPT_FASTOPEN_MAGIC)
					tcp_parse_fastopen_option(opsize -
						TCPOLEN_EXP_FASTOPEN_BASE,
						ptr + 2, th->syn, foc, true);
3861 3862 3863
				else
					smc_parse_options(th, opt_rx, ptr,
							  opsize);
Y
Yuchung Cheng 已提交
3864 3865 3866
				break;

			}
3867 3868
			ptr += opsize-2;
			length -= opsize;
3869
		}
L
Linus Torvalds 已提交
3870 3871
	}
}
E
Eric Dumazet 已提交
3872
EXPORT_SYMBOL(tcp_parse_options);
L
Linus Torvalds 已提交
3873

E
Eric Dumazet 已提交
3874
static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th)
3875
{
3876
	const __be32 *ptr = (const __be32 *)(th + 1);
3877 3878 3879 3880 3881 3882 3883

	if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
			  | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
		tp->rx_opt.saw_tstamp = 1;
		++ptr;
		tp->rx_opt.rcv_tsval = ntohl(*ptr);
		++ptr;
3884 3885 3886 3887
		if (*ptr)
			tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset;
		else
			tp->rx_opt.rcv_tsecr = 0;
E
Eric Dumazet 已提交
3888
		return true;
3889
	}
E
Eric Dumazet 已提交
3890
	return false;
3891 3892
}

L
Linus Torvalds 已提交
3893 3894 3895
/* Fast parse options. This hopes to only see timestamps.
 * If it is wrong it falls back on tcp_parse_options().
 */
3896 3897
static bool tcp_fast_parse_options(const struct net *net,
				   const struct sk_buff *skb,
C
Christoph Paasch 已提交
3898
				   const struct tcphdr *th, struct tcp_sock *tp)
L
Linus Torvalds 已提交
3899
{
3900 3901 3902 3903
	/* In the spirit of fast parsing, compare doff directly to constant
	 * values.  Because equality is used, short doff can be ignored here.
	 */
	if (th->doff == (sizeof(*th) / 4)) {
L
Linus Torvalds 已提交
3904
		tp->rx_opt.saw_tstamp = 0;
E
Eric Dumazet 已提交
3905
		return false;
L
Linus Torvalds 已提交
3906
	} else if (tp->rx_opt.tstamp_ok &&
3907
		   th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) {
3908
		if (tcp_parse_aligned_timestamp(tp, th))
E
Eric Dumazet 已提交
3909
			return true;
L
Linus Torvalds 已提交
3910
	}
3911

3912
	tcp_parse_options(net, skb, &tp->rx_opt, 1, NULL);
3913
	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
3914 3915
		tp->rx_opt.rcv_tsecr -= tp->tsoffset;

E
Eric Dumazet 已提交
3916
	return true;
L
Linus Torvalds 已提交
3917 3918
}

3919 3920 3921 3922
#ifdef CONFIG_TCP_MD5SIG
/*
 * Parse MD5 Signature option
 */
3923
const u8 *tcp_parse_md5sig_option(const struct tcphdr *th)
3924
{
3925 3926
	int length = (th->doff << 2) - sizeof(*th);
	const u8 *ptr = (const u8 *)(th + 1);
3927

J
Jann Horn 已提交
3928 3929
	/* If not enough data remaining, we can short cut */
	while (length >= TCPOLEN_MD5SIG) {
3930 3931 3932
		int opcode = *ptr++;
		int opsize;

W
Weilong Chen 已提交
3933
		switch (opcode) {
3934 3935 3936 3937 3938 3939 3940 3941 3942 3943
		case TCPOPT_EOL:
			return NULL;
		case TCPOPT_NOP:
			length--;
			continue;
		default:
			opsize = *ptr++;
			if (opsize < 2 || opsize > length)
				return NULL;
			if (opcode == TCPOPT_MD5SIG)
3944
				return opsize == TCPOLEN_MD5SIG ? ptr : NULL;
3945 3946 3947 3948 3949 3950
		}
		ptr += opsize - 2;
		length -= opsize;
	}
	return NULL;
}
E
Eric Dumazet 已提交
3951
EXPORT_SYMBOL(tcp_parse_md5sig_option);
3952 3953
#endif

L
Linus Torvalds 已提交
3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976
/* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM
 *
 * It is not fatal. If this ACK does _not_ change critical state (seqs, window)
 * it can pass through stack. So, the following predicate verifies that
 * this segment is not used for anything but congestion avoidance or
 * fast retransmit. Moreover, we even are able to eliminate most of such
 * second order effects, if we apply some small "replay" window (~RTO)
 * to timestamp space.
 *
 * All these measures still do not guarantee that we reject wrapped ACKs
 * on networks with high bandwidth, when sequence space is recycled fastly,
 * but it guarantees that such events will be very rare and do not affect
 * connection seriously. This doesn't look nice, but alas, PAWS is really
 * buggy extension.
 *
 * [ Later note. Even worse! It is buggy for segments _with_ data. RFC
 * states that events when retransmit arrives after original data are rare.
 * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is
 * the biggest problem on large power networks even with minor reordering.
 * OK, let's give it small replay window. If peer clock is even 1hz, it is safe
 * up to bandwidth of 18Gigabit/sec. 8) ]
 */

3977
static int tcp_disordered_ack(const struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
3978
{
3979 3980
	const struct tcp_sock *tp = tcp_sk(sk);
	const struct tcphdr *th = tcp_hdr(skb);
L
Linus Torvalds 已提交
3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993
	u32 seq = TCP_SKB_CB(skb)->seq;
	u32 ack = TCP_SKB_CB(skb)->ack_seq;

	return (/* 1. Pure ACK with correct sequence number. */
		(th->ack && seq == TCP_SKB_CB(skb)->end_seq && seq == tp->rcv_nxt) &&

		/* 2. ... and duplicate ACK. */
		ack == tp->snd_una &&

		/* 3. ... and does not update window. */
		!tcp_may_update_window(tp, ack, seq, ntohs(th->window) << tp->rx_opt.snd_wscale) &&

		/* 4. ... and sits in replay window. */
3994
		(s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) <= (inet_csk(sk)->icsk_rto * 1024) / HZ);
L
Linus Torvalds 已提交
3995 3996
}

3997
static inline bool tcp_paws_discard(const struct sock *sk,
3998
				   const struct sk_buff *skb)
L
Linus Torvalds 已提交
3999
{
4000
	const struct tcp_sock *tp = tcp_sk(sk);
I
Ilpo Järvinen 已提交
4001 4002 4003

	return !tcp_paws_check(&tp->rx_opt, TCP_PAWS_WINDOW) &&
	       !tcp_disordered_ack(sk, skb);
L
Linus Torvalds 已提交
4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018
}

/* Check segment sequence number for validity.
 *
 * Segment controls are considered valid, if the segment
 * fits to the window after truncation to the window. Acceptability
 * of data (and SYN, FIN, of course) is checked separately.
 * See tcp_data_queue(), for example.
 *
 * Also, controls (RST is main one) are accepted using RCV.WUP instead
 * of RCV.NXT. Peer still did not advance his SND.UNA when we
 * delayed ACK, so that hisSND.UNA<=ourRCV.WUP.
 * (borrowed from freebsd)
 */

4019
static inline bool tcp_sequence(const struct tcp_sock *tp, u32 seq, u32 end_seq)
L
Linus Torvalds 已提交
4020 4021 4022 4023 4024 4025
{
	return	!before(end_seq, tp->rcv_wup) &&
		!after(seq, tp->rcv_nxt + tcp_receive_window(tp));
}

/* When we get a reset we do this. */
4026
void tcp_reset(struct sock *sk)
L
Linus Torvalds 已提交
4027
{
4028 4029
	trace_tcp_receive_reset(sk);

L
Linus Torvalds 已提交
4030 4031
	/* We want the right error as BSD sees it (and indeed as we do). */
	switch (sk->sk_state) {
4032 4033 4034 4035 4036 4037 4038 4039 4040 4041
	case TCP_SYN_SENT:
		sk->sk_err = ECONNREFUSED;
		break;
	case TCP_CLOSE_WAIT:
		sk->sk_err = EPIPE;
		break;
	case TCP_CLOSE:
		return;
	default:
		sk->sk_err = ECONNRESET;
L
Linus Torvalds 已提交
4042
	}
T
Tom Marshall 已提交
4043 4044
	/* This barrier is coupled with smp_rmb() in tcp_poll() */
	smp_wmb();
L
Linus Torvalds 已提交
4045

4046
	tcp_write_queue_purge(sk);
4047 4048
	tcp_done(sk);

L
Linus Torvalds 已提交
4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066
	if (!sock_flag(sk, SOCK_DEAD))
		sk->sk_error_report(sk);
}

/*
 * 	Process the FIN bit. This now behaves as it is supposed to work
 *	and the FIN takes effect when it is validly part of sequence
 *	space. Not before when we get holes.
 *
 *	If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT
 *	(and thence onto LAST-ACK and finally, CLOSE, we never enter
 *	TIME-WAIT)
 *
 *	If we are in FINWAIT-1, a received FIN indicates simultaneous
 *	close and we go into CLOSING (and later onto TIME-WAIT)
 *
 *	If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT.
 */
4067
void tcp_fin(struct sock *sk)
L
Linus Torvalds 已提交
4068 4069 4070
{
	struct tcp_sock *tp = tcp_sk(sk);

4071
	inet_csk_schedule_ack(sk);
L
Linus Torvalds 已提交
4072 4073 4074 4075 4076

	sk->sk_shutdown |= RCV_SHUTDOWN;
	sock_set_flag(sk, SOCK_DONE);

	switch (sk->sk_state) {
4077 4078 4079 4080
	case TCP_SYN_RECV:
	case TCP_ESTABLISHED:
		/* Move to CLOSE_WAIT */
		tcp_set_state(sk, TCP_CLOSE_WAIT);
4081
		inet_csk(sk)->icsk_ack.pingpong = 1;
4082
		break;
L
Linus Torvalds 已提交
4083

4084 4085 4086 4087 4088 4089 4090 4091 4092
	case TCP_CLOSE_WAIT:
	case TCP_CLOSING:
		/* Received a retransmission of the FIN, do
		 * nothing.
		 */
		break;
	case TCP_LAST_ACK:
		/* RFC793: Remain in the LAST-ACK state. */
		break;
L
Linus Torvalds 已提交
4093

4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110
	case TCP_FIN_WAIT1:
		/* This case occurs when a simultaneous close
		 * happens, we must ack the received FIN and
		 * enter the CLOSING state.
		 */
		tcp_send_ack(sk);
		tcp_set_state(sk, TCP_CLOSING);
		break;
	case TCP_FIN_WAIT2:
		/* Received a FIN -- send ACK and enter TIME_WAIT. */
		tcp_send_ack(sk);
		tcp_time_wait(sk, TCP_TIME_WAIT, 0);
		break;
	default:
		/* Only TCP_LISTEN and TCP_CLOSE are left, in these
		 * cases we should never reach this piece of code.
		 */
J
Joe Perches 已提交
4111
		pr_err("%s: Impossible, sk->sk_state=%d\n",
4112
		       __func__, sk->sk_state);
4113
		break;
4114
	}
L
Linus Torvalds 已提交
4115 4116 4117 4118

	/* It _is_ possible, that we have something out-of-order _after_ FIN.
	 * Probably, we should reset in this case. For now drop them.
	 */
4119
	skb_rbtree_purge(&tp->out_of_order_queue);
4120
	if (tcp_is_sack(tp))
L
Linus Torvalds 已提交
4121
		tcp_sack_reset(&tp->rx_opt);
4122
	sk_mem_reclaim(sk);
L
Linus Torvalds 已提交
4123 4124 4125 4126 4127 4128 4129

	if (!sock_flag(sk, SOCK_DEAD)) {
		sk->sk_state_change(sk);

		/* Do not send POLL_HUP for half duplex close. */
		if (sk->sk_shutdown == SHUTDOWN_MASK ||
		    sk->sk_state == TCP_CLOSE)
4130
			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
L
Linus Torvalds 已提交
4131
		else
4132
			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
L
Linus Torvalds 已提交
4133 4134 4135
	}
}

E
Eric Dumazet 已提交
4136
static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq,
4137
				  u32 end_seq)
L
Linus Torvalds 已提交
4138 4139 4140 4141 4142 4143
{
	if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) {
		if (before(seq, sp->start_seq))
			sp->start_seq = seq;
		if (after(end_seq, sp->end_seq))
			sp->end_seq = end_seq;
E
Eric Dumazet 已提交
4144
		return true;
L
Linus Torvalds 已提交
4145
	}
E
Eric Dumazet 已提交
4146
	return false;
L
Linus Torvalds 已提交
4147 4148
}

4149
static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq)
L
Linus Torvalds 已提交
4150
{
4151 4152
	struct tcp_sock *tp = tcp_sk(sk);

E
Eric Dumazet 已提交
4153
	if (tcp_is_sack(tp) && sock_net(sk)->ipv4.sysctl_tcp_dsack) {
4154 4155
		int mib_idx;

L
Linus Torvalds 已提交
4156
		if (before(seq, tp->rcv_nxt))
4157
			mib_idx = LINUX_MIB_TCPDSACKOLDSENT;
L
Linus Torvalds 已提交
4158
		else
4159 4160
			mib_idx = LINUX_MIB_TCPDSACKOFOSENT;

4161
		NET_INC_STATS(sock_net(sk), mib_idx);
L
Linus Torvalds 已提交
4162 4163 4164 4165 4166 4167 4168

		tp->rx_opt.dsack = 1;
		tp->duplicate_sack[0].start_seq = seq;
		tp->duplicate_sack[0].end_seq = end_seq;
	}
}

4169
static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq)
L
Linus Torvalds 已提交
4170
{
4171 4172
	struct tcp_sock *tp = tcp_sk(sk);

L
Linus Torvalds 已提交
4173
	if (!tp->rx_opt.dsack)
4174
		tcp_dsack_set(sk, seq, end_seq);
L
Linus Torvalds 已提交
4175 4176 4177 4178
	else
		tcp_sack_extend(tp->duplicate_sack, seq, end_seq);
}

4179
static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb)
L
Linus Torvalds 已提交
4180 4181 4182 4183 4184
{
	struct tcp_sock *tp = tcp_sk(sk);

	if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
	    before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
4185
		NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
4186
		tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
L
Linus Torvalds 已提交
4187

E
Eric Dumazet 已提交
4188
		if (tcp_is_sack(tp) && sock_net(sk)->ipv4.sysctl_tcp_dsack) {
L
Linus Torvalds 已提交
4189 4190 4191 4192
			u32 end_seq = TCP_SKB_CB(skb)->end_seq;

			if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))
				end_seq = tp->rcv_nxt;
4193
			tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq);
L
Linus Torvalds 已提交
4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206
		}
	}

	tcp_send_ack(sk);
}

/* These routines update the SACK block as out-of-order packets arrive or
 * in-order packets close up the sequence space.
 */
static void tcp_sack_maybe_coalesce(struct tcp_sock *tp)
{
	int this_sack;
	struct tcp_sack_block *sp = &tp->selective_acks[0];
4207
	struct tcp_sack_block *swalk = sp + 1;
L
Linus Torvalds 已提交
4208 4209 4210 4211

	/* See if the recent change to the first SACK eats into
	 * or hits the sequence space of other SACK blocks, if so coalesce.
	 */
4212
	for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) {
L
Linus Torvalds 已提交
4213 4214 4215 4216 4217 4218 4219
		if (tcp_sack_extend(sp, swalk->start_seq, swalk->end_seq)) {
			int i;

			/* Zap SWALK, by moving every further SACK up by one slot.
			 * Decrease num_sacks.
			 */
			tp->rx_opt.num_sacks--;
4220 4221
			for (i = this_sack; i < tp->rx_opt.num_sacks; i++)
				sp[i] = sp[i + 1];
L
Linus Torvalds 已提交
4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237
			continue;
		}
		this_sack++, swalk++;
	}
}

static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq)
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct tcp_sack_block *sp = &tp->selective_acks[0];
	int cur_sacks = tp->rx_opt.num_sacks;
	int this_sack;

	if (!cur_sacks)
		goto new_sack;

4238
	for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) {
L
Linus Torvalds 已提交
4239 4240
		if (tcp_sack_extend(sp, seq, end_seq)) {
			/* Rotate this_sack to the first one. */
4241
			for (; this_sack > 0; this_sack--, sp--)
4242
				swap(*sp, *(sp - 1));
L
Linus Torvalds 已提交
4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254
			if (cur_sacks > 1)
				tcp_sack_maybe_coalesce(tp);
			return;
		}
	}

	/* Could not find an adjacent existing SACK, build a new one,
	 * put it at the front, and shift everyone else down.  We
	 * always know there is at least one SACK present already here.
	 *
	 * If the sack array is full, forget about the last one.
	 */
4255
	if (this_sack >= TCP_NUM_SACKS) {
E
Eric Dumazet 已提交
4256 4257
		if (tp->compressed_ack)
			tcp_send_ack(sk);
L
Linus Torvalds 已提交
4258 4259 4260 4261
		this_sack--;
		tp->rx_opt.num_sacks--;
		sp--;
	}
S
Stephen Hemminger 已提交
4262
	for (; this_sack > 0; this_sack--, sp--)
4263
		*sp = *(sp - 1);
L
Linus Torvalds 已提交
4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280

new_sack:
	/* Build the new head SACK, and we're done. */
	sp->start_seq = seq;
	sp->end_seq = end_seq;
	tp->rx_opt.num_sacks++;
}

/* RCV.NXT advances, some SACKs should be eaten. */

static void tcp_sack_remove(struct tcp_sock *tp)
{
	struct tcp_sack_block *sp = &tp->selective_acks[0];
	int num_sacks = tp->rx_opt.num_sacks;
	int this_sack;

	/* Empty ofo queue, hence, all the SACKs are eaten. Clear. */
4281
	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
L
Linus Torvalds 已提交
4282 4283 4284 4285
		tp->rx_opt.num_sacks = 0;
		return;
	}

4286
	for (this_sack = 0; this_sack < num_sacks;) {
L
Linus Torvalds 已提交
4287 4288 4289 4290 4291
		/* Check if the start of the sack is covered by RCV.NXT. */
		if (!before(tp->rcv_nxt, sp->start_seq)) {
			int i;

			/* RCV.NXT must cover all the block! */
4292
			WARN_ON(before(tp->rcv_nxt, sp->end_seq));
L
Linus Torvalds 已提交
4293 4294

			/* Zap this SACK, by moving forward any other SACKS. */
W
Weilong Chen 已提交
4295
			for (i = this_sack+1; i < num_sacks; i++)
L
Linus Torvalds 已提交
4296 4297 4298 4299 4300 4301 4302
				tp->selective_acks[i-1] = tp->selective_acks[i];
			num_sacks--;
			continue;
		}
		this_sack++;
		sp++;
	}
4303
	tp->rx_opt.num_sacks = num_sacks;
L
Linus Torvalds 已提交
4304 4305
}

4306 4307 4308
/**
 * tcp_try_coalesce - try to merge skb to prior one
 * @sk: socket
4309
 * @dest: destination queue
4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337
 * @to: prior buffer
 * @from: buffer to add in queue
 * @fragstolen: pointer to boolean
 *
 * Before queueing skb @from after @to, try to merge them
 * to reduce overall memory use and queue lengths, if cost is small.
 * Packets in ofo or receive queues can stay a long time.
 * Better try to coalesce them right now to avoid future collapses.
 * Returns true if caller should free @from instead of queueing it
 */
static bool tcp_try_coalesce(struct sock *sk,
			     struct sk_buff *to,
			     struct sk_buff *from,
			     bool *fragstolen)
{
	int delta;

	*fragstolen = false;

	/* Its possible this segment overlaps with prior segment in queue */
	if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq)
		return false;

	if (!skb_try_coalesce(to, from, fragstolen, &delta))
		return false;

	atomic_add(delta, &sk->sk_rmem_alloc);
	sk_mem_charge(sk, delta);
4338
	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE);
4339 4340 4341
	TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq;
	TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq;
	TCP_SKB_CB(to)->tcp_flags |= TCP_SKB_CB(from)->tcp_flags;
4342 4343 4344

	if (TCP_SKB_CB(from)->has_rxtstamp) {
		TCP_SKB_CB(to)->has_rxtstamp = true;
E
Eric Dumazet 已提交
4345
		to->tstamp = from->tstamp;
4346 4347
	}

4348 4349 4350
	return true;
}

4351 4352 4353 4354 4355 4356
static void tcp_drop(struct sock *sk, struct sk_buff *skb)
{
	sk_drops_add(sk, skb);
	__kfree_skb(skb);
}

L
Linus Torvalds 已提交
4357 4358 4359 4360 4361 4362 4363
/* This one checks to see if we can put data from the
 * out_of_order queue into the receive_queue.
 */
static void tcp_ofo_queue(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
	__u32 dsack_high = tp->rcv_nxt;
4364
	bool fin, fragstolen, eaten;
4365
	struct sk_buff *skb, *tail;
4366
	struct rb_node *p;
L
Linus Torvalds 已提交
4367

4368 4369
	p = rb_first(&tp->out_of_order_queue);
	while (p) {
4370
		skb = rb_to_skb(p);
L
Linus Torvalds 已提交
4371 4372 4373 4374 4375 4376 4377
		if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
			break;

		if (before(TCP_SKB_CB(skb)->seq, dsack_high)) {
			__u32 dsack = dsack_high;
			if (before(TCP_SKB_CB(skb)->end_seq, dsack_high))
				dsack_high = TCP_SKB_CB(skb)->end_seq;
4378
			tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack);
L
Linus Torvalds 已提交
4379
		}
4380 4381
		p = rb_next(p);
		rb_erase(&skb->rbnode, &tp->out_of_order_queue);
L
Linus Torvalds 已提交
4382

4383
		if (unlikely(!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))) {
4384
			SOCK_DEBUG(sk, "ofo packet was already received\n");
4385
			tcp_drop(sk, skb);
L
Linus Torvalds 已提交
4386 4387 4388 4389 4390 4391
			continue;
		}
		SOCK_DEBUG(sk, "ofo requeuing : rcv_next %X seq %X - %X\n",
			   tp->rcv_nxt, TCP_SKB_CB(skb)->seq,
			   TCP_SKB_CB(skb)->end_seq);

4392
		tail = skb_peek_tail(&sk->sk_receive_queue);
E
Eric Dumazet 已提交
4393
		eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen);
4394
		tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq);
4395
		fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
4396 4397
		if (!eaten)
			__skb_queue_tail(&sk->sk_receive_queue, skb);
4398
		else
4399
			kfree_skb_partial(skb, fragstolen);
4400 4401 4402 4403 4404 4405 4406 4407

		if (unlikely(fin)) {
			tcp_fin(sk);
			/* tcp_fin() purges tp->out_of_order_queue,
			 * so we must end this loop right now.
			 */
			break;
		}
L
Linus Torvalds 已提交
4408 4409 4410
	}
}

E
Eric Dumazet 已提交
4411
static bool tcp_prune_ofo_queue(struct sock *sk);
L
Linus Torvalds 已提交
4412 4413
static int tcp_prune_queue(struct sock *sk);

4414 4415
static int tcp_try_rmem_schedule(struct sock *sk, struct sk_buff *skb,
				 unsigned int size)
4416 4417
{
	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
4418
	    !sk_rmem_schedule(sk, skb, size)) {
4419 4420 4421 4422

		if (tcp_prune_queue(sk) < 0)
			return -1;

4423
		while (!sk_rmem_schedule(sk, skb, size)) {
4424 4425
			if (!tcp_prune_ofo_queue(sk))
				return -1;
4426 4427 4428 4429 4430
		}
	}
	return 0;
}

E
Eric Dumazet 已提交
4431 4432 4433
static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);
4434
	struct rb_node **p, *parent;
E
Eric Dumazet 已提交
4435 4436
	struct sk_buff *skb1;
	u32 seq, end_seq;
4437
	bool fragstolen;
E
Eric Dumazet 已提交
4438

4439
	tcp_ecn_check_ce(sk, skb);
E
Eric Dumazet 已提交
4440

4441
	if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) {
4442
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP);
4443
		tcp_drop(sk, skb);
E
Eric Dumazet 已提交
4444 4445 4446
		return;
	}

4447 4448
	/* Disable header prediction. */
	tp->pred_flags = 0;
E
Eric Dumazet 已提交
4449 4450
	inet_csk_schedule_ack(sk);

4451
	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOQUEUE);
4452 4453
	seq = TCP_SKB_CB(skb)->seq;
	end_seq = TCP_SKB_CB(skb)->end_seq;
E
Eric Dumazet 已提交
4454
	SOCK_DEBUG(sk, "out of order segment: rcv_next %X seq %X - %X\n",
4455
		   tp->rcv_nxt, seq, end_seq);
E
Eric Dumazet 已提交
4456

4457 4458
	p = &tp->out_of_order_queue.rb_node;
	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
E
Eric Dumazet 已提交
4459 4460 4461
		/* Initial out of order segment, build 1 SACK. */
		if (tcp_is_sack(tp)) {
			tp->rx_opt.num_sacks = 1;
4462 4463
			tp->selective_acks[0].start_seq = seq;
			tp->selective_acks[0].end_seq = end_seq;
E
Eric Dumazet 已提交
4464
		}
4465 4466 4467
		rb_link_node(&skb->rbnode, NULL, p);
		rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
		tp->ooo_last_skb = skb;
E
Eric Dumazet 已提交
4468 4469 4470
		goto end;
	}

4471 4472 4473
	/* In the typical case, we are adding an skb to the end of the list.
	 * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
	 */
E
Eric Dumazet 已提交
4474
	if (tcp_try_coalesce(sk, tp->ooo_last_skb,
4475
			     skb, &fragstolen)) {
4476 4477 4478 4479 4480 4481
coalesce_done:
		tcp_grow_window(sk, skb);
		kfree_skb_partial(skb, fragstolen);
		skb = NULL;
		goto add_sack;
	}
4482 4483 4484 4485 4486 4487
	/* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
	if (!before(seq, TCP_SKB_CB(tp->ooo_last_skb)->end_seq)) {
		parent = &tp->ooo_last_skb->rbnode;
		p = &parent->rb_right;
		goto insert;
	}
4488 4489 4490 4491 4492

	/* Find place to insert this segment. Handle overlaps on the way. */
	parent = NULL;
	while (*p) {
		parent = *p;
4493
		skb1 = rb_to_skb(parent);
4494 4495 4496
		if (before(seq, TCP_SKB_CB(skb1)->seq)) {
			p = &parent->rb_left;
			continue;
E
Eric Dumazet 已提交
4497
		}
4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522
		if (before(seq, TCP_SKB_CB(skb1)->end_seq)) {
			if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
				/* All the bits are present. Drop. */
				NET_INC_STATS(sock_net(sk),
					      LINUX_MIB_TCPOFOMERGE);
				__kfree_skb(skb);
				skb = NULL;
				tcp_dsack_set(sk, seq, end_seq);
				goto add_sack;
			}
			if (after(seq, TCP_SKB_CB(skb1)->seq)) {
				/* Partial overlap. */
				tcp_dsack_set(sk, seq, TCP_SKB_CB(skb1)->end_seq);
			} else {
				/* skb's seq == skb1's seq and skb covers skb1.
				 * Replace skb1 with skb.
				 */
				rb_replace_node(&skb1->rbnode, &skb->rbnode,
						&tp->out_of_order_queue);
				tcp_dsack_extend(sk,
						 TCP_SKB_CB(skb1)->seq,
						 TCP_SKB_CB(skb1)->end_seq);
				NET_INC_STATS(sock_net(sk),
					      LINUX_MIB_TCPOFOMERGE);
				__kfree_skb(skb1);
4523
				goto merge_right;
4524
			}
E
Eric Dumazet 已提交
4525
		} else if (tcp_try_coalesce(sk, skb1,
4526
					    skb, &fragstolen)) {
4527
			goto coalesce_done;
E
Eric Dumazet 已提交
4528
		}
4529
		p = &parent->rb_right;
E
Eric Dumazet 已提交
4530
	}
4531
insert:
4532 4533 4534
	/* Insert segment into RB tree. */
	rb_link_node(&skb->rbnode, parent, p);
	rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
E
Eric Dumazet 已提交
4535

4536
merge_right:
4537
	/* Remove other segments covered by skb. */
4538
	while ((skb1 = skb_rb_next(skb)) != NULL) {
E
Eric Dumazet 已提交
4539 4540 4541 4542 4543 4544 4545
		if (!after(end_seq, TCP_SKB_CB(skb1)->seq))
			break;
		if (before(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
			tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
					 end_seq);
			break;
		}
4546
		rb_erase(&skb1->rbnode, &tp->out_of_order_queue);
E
Eric Dumazet 已提交
4547 4548
		tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
				 TCP_SKB_CB(skb1)->end_seq);
4549
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
4550
		tcp_drop(sk, skb1);
E
Eric Dumazet 已提交
4551
	}
4552
	/* If there is no skb after us, we are the last_skb ! */
4553
	if (!skb1)
4554
		tp->ooo_last_skb = skb;
E
Eric Dumazet 已提交
4555 4556 4557 4558 4559

add_sack:
	if (tcp_is_sack(tp))
		tcp_sack_new_ofo_skb(sk, seq, end_seq);
end:
4560 4561
	if (skb) {
		tcp_grow_window(sk, skb);
4562
		skb_condense(skb);
E
Eric Dumazet 已提交
4563
		skb_set_owner_r(skb, sk);
4564
	}
E
Eric Dumazet 已提交
4565 4566
}

4567
static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int hdrlen,
4568 4569 4570 4571 4572 4573 4574
		  bool *fragstolen)
{
	int eaten;
	struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue);

	__skb_pull(skb, hdrlen);
	eaten = (tail &&
E
Eric Dumazet 已提交
4575
		 tcp_try_coalesce(sk, tail,
4576
				  skb, fragstolen)) ? 1 : 0;
4577
	tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq);
4578 4579 4580 4581 4582 4583
	if (!eaten) {
		__skb_queue_tail(&sk->sk_receive_queue, skb);
		skb_set_owner_r(skb, sk);
	}
	return eaten;
}
E
Eric Dumazet 已提交
4584

4585 4586
int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)
{
4587
	struct sk_buff *skb;
4588 4589
	int err = -ENOMEM;
	int data_len = 0;
4590 4591
	bool fragstolen;

4592 4593 4594
	if (size == 0)
		return 0;

4595 4596 4597 4598 4599 4600 4601 4602 4603
	if (size > PAGE_SIZE) {
		int npages = min_t(size_t, size >> PAGE_SHIFT, MAX_SKB_FRAGS);

		data_len = npages << PAGE_SHIFT;
		size = data_len + (size & ~PAGE_MASK);
	}
	skb = alloc_skb_with_frags(size - data_len, data_len,
				   PAGE_ALLOC_COSTLY_ORDER,
				   &err, sk->sk_allocation);
4604 4605 4606
	if (!skb)
		goto err;

4607 4608 4609 4610
	skb_put(skb, size - data_len);
	skb->data_len = data_len;
	skb->len = size;

4611
	if (tcp_try_rmem_schedule(sk, skb, skb->truesize))
4612 4613
		goto err_free;

4614 4615
	err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size);
	if (err)
4616 4617 4618 4619 4620 4621
		goto err_free;

	TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt;
	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size;
	TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1;

4622
	if (tcp_queue_rcv(sk, skb, 0, &fragstolen)) {
4623 4624 4625 4626 4627 4628 4629 4630
		WARN_ON_ONCE(fragstolen); /* should not happen */
		__kfree_skb(skb);
	}
	return size;

err_free:
	kfree_skb(skb);
err:
4631 4632
	return err;

4633 4634
}

4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645
void tcp_data_ready(struct sock *sk)
{
	const struct tcp_sock *tp = tcp_sk(sk);
	int avail = tp->rcv_nxt - tp->copied_seq;

	if (avail < sk->sk_rcvlowat && !sock_flag(sk, SOCK_DONE))
		return;

	sk->sk_data_ready(sk);
}

L
Linus Torvalds 已提交
4646 4647 4648
static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);
E
Eric Dumazet 已提交
4649 4650
	bool fragstolen;
	int eaten;
L
Linus Torvalds 已提交
4651

4652 4653 4654 4655
	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
		__kfree_skb(skb);
		return;
	}
E
Eric Dumazet 已提交
4656
	skb_dst_drop(skb);
4657
	__skb_pull(skb, tcp_hdr(skb)->doff * 4);
L
Linus Torvalds 已提交
4658

4659
	tcp_ecn_accept_cwr(tp, skb);
L
Linus Torvalds 已提交
4660

4661
	tp->rx_opt.dsack = 0;
L
Linus Torvalds 已提交
4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672

	/*  Queue data for delivery to the user.
	 *  Packets in sequence go to the receive queue.
	 *  Out of sequence packets to the out_of_order_queue.
	 */
	if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) {
		if (tcp_receive_window(tp) == 0)
			goto out_of_window;

		/* Ok. In sequence. In window. */
queue_and_out:
E
Eric Dumazet 已提交
4673 4674 4675 4676 4677
		if (skb_queue_len(&sk->sk_receive_queue) == 0)
			sk_forced_mem_schedule(sk, skb->truesize);
		else if (tcp_try_rmem_schedule(sk, skb, skb->truesize))
			goto drop;

4678
		eaten = tcp_queue_rcv(sk, skb, 0, &fragstolen);
4679
		tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq);
S
Stephen Hemminger 已提交
4680
		if (skb->len)
4681
			tcp_event_data_recv(sk, skb);
4682
		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
4683
			tcp_fin(sk);
L
Linus Torvalds 已提交
4684

4685
		if (!RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
L
Linus Torvalds 已提交
4686 4687 4688 4689 4690
			tcp_ofo_queue(sk);

			/* RFC2581. 4.2. SHOULD send immediate ACK, when
			 * gap in queue is filled.
			 */
4691
			if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
4692
				inet_csk(sk)->icsk_ack.pingpong = 0;
L
Linus Torvalds 已提交
4693 4694 4695 4696 4697
		}

		if (tp->rx_opt.num_sacks)
			tcp_sack_remove(tp);

4698 4699
		tcp_fast_path_check(sk);

4700 4701
		if (eaten > 0)
			kfree_skb_partial(skb, fragstolen);
4702
		if (!sock_flag(sk, SOCK_DEAD))
4703
			tcp_data_ready(sk);
L
Linus Torvalds 已提交
4704 4705 4706 4707 4708
		return;
	}

	if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
		/* A retransmit, 2nd most common case.  Force an immediate ack. */
4709
		NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
4710
		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
L
Linus Torvalds 已提交
4711 4712

out_of_window:
4713
		tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
4714
		inet_csk_schedule_ack(sk);
L
Linus Torvalds 已提交
4715
drop:
4716
		tcp_drop(sk, skb);
L
Linus Torvalds 已提交
4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729
		return;
	}

	/* Out of window. F.e. zero window probe. */
	if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt + tcp_receive_window(tp)))
		goto out_of_window;

	if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
		/* Partial packet, seq < rcv_next < end_seq */
		SOCK_DEBUG(sk, "partial packet: rcv_next %X seq %X - %X\n",
			   tp->rcv_nxt, TCP_SKB_CB(skb)->seq,
			   TCP_SKB_CB(skb)->end_seq);

4730
		tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, tp->rcv_nxt);
4731

L
Linus Torvalds 已提交
4732 4733 4734 4735 4736 4737 4738 4739
		/* If window is closed, drop tail of packet. But after
		 * remembering D-SACK for its head made in previous line.
		 */
		if (!tcp_receive_window(tp))
			goto out_of_window;
		goto queue_and_out;
	}

E
Eric Dumazet 已提交
4740
	tcp_data_queue_ofo(sk, skb);
L
Linus Torvalds 已提交
4741 4742
}

4743 4744 4745 4746 4747
static struct sk_buff *tcp_skb_next(struct sk_buff *skb, struct sk_buff_head *list)
{
	if (list)
		return !skb_queue_is_last(list, skb) ? skb->next : NULL;

4748
	return skb_rb_next(skb);
4749 4750
}

4751
static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb,
4752 4753
					struct sk_buff_head *list,
					struct rb_root *root)
4754
{
4755
	struct sk_buff *next = tcp_skb_next(skb, list);
4756

4757 4758 4759 4760
	if (list)
		__skb_unlink(skb, list);
	else
		rb_erase(&skb->rbnode, root);
4761 4762

	__kfree_skb(skb);
4763
	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED);
4764 4765 4766 4767

	return next;
}

4768
/* Insert skb into rb tree, ordered by TCP_SKB_CB(skb)->seq */
4769
void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb)
4770 4771 4772 4773 4774 4775 4776
{
	struct rb_node **p = &root->rb_node;
	struct rb_node *parent = NULL;
	struct sk_buff *skb1;

	while (*p) {
		parent = *p;
4777
		skb1 = rb_to_skb(parent);
4778 4779 4780 4781 4782 4783 4784 4785 4786
		if (before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb1)->seq))
			p = &parent->rb_left;
		else
			p = &parent->rb_right;
	}
	rb_link_node(&skb->rbnode, parent, p);
	rb_insert_color(&skb->rbnode, root);
}

L
Linus Torvalds 已提交
4787 4788
/* Collapse contiguous sequence of skbs head..tail with
 * sequence numbers start..end.
4789
 *
4790
 * If tail is NULL, this means until the end of the queue.
4791
 *
L
Linus Torvalds 已提交
4792 4793 4794 4795
 * Segments with FIN/SYN are not collapsed (only because this
 * simplifies code)
 */
static void
4796 4797
tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
	     struct sk_buff *head, struct sk_buff *tail, u32 start, u32 end)
L
Linus Torvalds 已提交
4798
{
4799 4800
	struct sk_buff *skb = head, *n;
	struct sk_buff_head tmp;
4801
	bool end_of_skbs;
L
Linus Torvalds 已提交
4802

S
Stephen Hemminger 已提交
4803
	/* First, check that queue is collapsible and find
4804 4805
	 * the point where collapsing can be useful.
	 */
4806
restart:
4807 4808 4809
	for (end_of_skbs = true; skb != NULL && skb != tail; skb = n) {
		n = tcp_skb_next(skb, list);

L
Linus Torvalds 已提交
4810 4811
		/* No new bits? It is possible on ofo queue. */
		if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
4812
			skb = tcp_collapse_one(sk, skb, list, root);
4813 4814 4815
			if (!skb)
				break;
			goto restart;
L
Linus Torvalds 已提交
4816 4817 4818 4819 4820 4821 4822
		}

		/* The first skb to collapse is:
		 * - not SYN/FIN and
		 * - bloated or contains data before "start" or
		 *   overlaps to the next one.
		 */
4823
		if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&
4824
		    (tcp_win_from_space(sk, skb->truesize) > skb->len ||
4825 4826
		     before(TCP_SKB_CB(skb)->seq, start))) {
			end_of_skbs = false;
L
Linus Torvalds 已提交
4827
			break;
4828 4829
		}

4830 4831 4832 4833
		if (n && n != tail &&
		    TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(n)->seq) {
			end_of_skbs = false;
			break;
4834
		}
L
Linus Torvalds 已提交
4835 4836 4837 4838

		/* Decided to skip this, advance start seq. */
		start = TCP_SKB_CB(skb)->end_seq;
	}
4839 4840
	if (end_of_skbs ||
	    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
L
Linus Torvalds 已提交
4841 4842
		return;

4843 4844
	__skb_queue_head_init(&tmp);

L
Linus Torvalds 已提交
4845
	while (before(start, end)) {
4846
		int copy = min_t(int, SKB_MAX_ORDER(0, 0), end - start);
L
Linus Torvalds 已提交
4847 4848
		struct sk_buff *nskb;

4849
		nskb = alloc_skb(copy, GFP_ATOMIC);
L
Linus Torvalds 已提交
4850
		if (!nskb)
4851
			break;
4852

L
Linus Torvalds 已提交
4853 4854
		memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
		TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
4855 4856 4857 4858
		if (list)
			__skb_queue_before(list, skb, nskb);
		else
			__skb_queue_tail(&tmp, nskb); /* defer rbtree insertion */
4859
		skb_set_owner_r(nskb, sk);
L
Linus Torvalds 已提交
4860 4861 4862 4863 4864 4865

		/* Copy data, releasing collapsed skbs. */
		while (copy > 0) {
			int offset = start - TCP_SKB_CB(skb)->seq;
			int size = TCP_SKB_CB(skb)->end_seq - start;

4866
			BUG_ON(offset < 0);
L
Linus Torvalds 已提交
4867 4868 4869 4870 4871 4872 4873 4874 4875
			if (size > 0) {
				size = min(copy, size);
				if (skb_copy_bits(skb, offset, skb_put(nskb, size), size))
					BUG();
				TCP_SKB_CB(nskb)->end_seq += size;
				copy -= size;
				start += size;
			}
			if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
4876
				skb = tcp_collapse_one(sk, skb, list, root);
4877 4878
				if (!skb ||
				    skb == tail ||
4879
				    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
4880
					goto end;
L
Linus Torvalds 已提交
4881 4882 4883
			}
		}
	}
4884 4885 4886
end:
	skb_queue_walk_safe(&tmp, skb, n)
		tcp_rbtree_insert(root, skb);
L
Linus Torvalds 已提交
4887 4888 4889 4890 4891 4892 4893 4894
}

/* Collapse ofo queue. Algorithm: select contiguous sequence of skbs
 * and tcp_collapse() them until all the queue is collapsed.
 */
static void tcp_collapse_ofo_queue(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);
4895
	struct sk_buff *skb, *head;
L
Linus Torvalds 已提交
4896 4897
	u32 start, end;

4898
	skb = skb_rb_first(&tp->out_of_order_queue);
4899 4900
new_range:
	if (!skb) {
4901
		tp->ooo_last_skb = skb_rb_last(&tp->out_of_order_queue);
L
Linus Torvalds 已提交
4902
		return;
4903
	}
L
Linus Torvalds 已提交
4904 4905
	start = TCP_SKB_CB(skb)->seq;
	end = TCP_SKB_CB(skb)->end_seq;
4906

4907
	for (head = skb;;) {
4908
		skb = skb_rb_next(skb);
L
Linus Torvalds 已提交
4909

4910 4911 4912
		/* Range is terminated when we see a gap or when
		 * we are at the queue end.
		 */
4913
		if (!skb ||
L
Linus Torvalds 已提交
4914 4915
		    after(TCP_SKB_CB(skb)->seq, end) ||
		    before(TCP_SKB_CB(skb)->end_seq, start)) {
4916
			tcp_collapse(sk, NULL, &tp->out_of_order_queue,
D
David S. Miller 已提交
4917
				     head, skb, start, end);
4918 4919 4920 4921
			goto new_range;
		}

		if (unlikely(before(TCP_SKB_CB(skb)->seq, start)))
L
Linus Torvalds 已提交
4922
			start = TCP_SKB_CB(skb)->seq;
4923
		if (after(TCP_SKB_CB(skb)->end_seq, end))
L
Linus Torvalds 已提交
4924 4925 4926 4927
			end = TCP_SKB_CB(skb)->end_seq;
	}
}

4928
/*
4929 4930 4931 4932 4933 4934 4935 4936
 * Clean the out-of-order queue to make room.
 * We drop high sequences packets to :
 * 1) Let a chance for holes to be filled.
 * 2) not add too big latencies if thousands of packets sit there.
 *    (But if application shrinks SO_RCVBUF, we could still end up
 *     freeing whole queue here)
 *
 * Return true if queue has shrunk.
4937
 */
E
Eric Dumazet 已提交
4938
static bool tcp_prune_ofo_queue(struct sock *sk)
4939 4940
{
	struct tcp_sock *tp = tcp_sk(sk);
4941
	struct rb_node *node, *prev;
4942

4943
	if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
4944
		return false;
4945

4946
	NET_INC_STATS(sock_net(sk), LINUX_MIB_OFOPRUNED);
4947 4948 4949 4950
	node = &tp->ooo_last_skb->rbnode;
	do {
		prev = rb_prev(node);
		rb_erase(node, &tp->out_of_order_queue);
4951
		tcp_drop(sk, rb_to_skb(node));
4952
		sk_mem_reclaim(sk);
4953 4954 4955
		if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf &&
		    !tcp_under_memory_pressure(sk))
			break;
4956 4957
		node = prev;
	} while (node);
4958
	tp->ooo_last_skb = rb_to_skb(prev);
4959 4960 4961 4962 4963 4964 4965 4966 4967

	/* Reset SACK state.  A conforming SACK implementation will
	 * do the same at a timeout based retransmit.  When a connection
	 * is in a sad state like this, we care only about integrity
	 * of the connection not performance.
	 */
	if (tp->rx_opt.sack_ok)
		tcp_sack_reset(&tp->rx_opt);
	return true;
4968 4969
}

L
Linus Torvalds 已提交
4970 4971 4972 4973 4974 4975 4976 4977 4978
/* Reduce allocated memory if we can, trying to get
 * the socket within its memory limits again.
 *
 * Return less than zero if we should start dropping frames
 * until the socket owning process reads some of the data
 * to stabilize the situation.
 */
static int tcp_prune_queue(struct sock *sk)
{
4979
	struct tcp_sock *tp = tcp_sk(sk);
L
Linus Torvalds 已提交
4980 4981 4982

	SOCK_DEBUG(sk, "prune_queue: c=%x\n", tp->copied_seq);

4983
	NET_INC_STATS(sock_net(sk), LINUX_MIB_PRUNECALLED);
L
Linus Torvalds 已提交
4984 4985

	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
4986
		tcp_clamp_window(sk);
4987
	else if (tcp_under_memory_pressure(sk))
L
Linus Torvalds 已提交
4988 4989 4990
		tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss);

	tcp_collapse_ofo_queue(sk);
4991
	if (!skb_queue_empty(&sk->sk_receive_queue))
4992
		tcp_collapse(sk, &sk->sk_receive_queue, NULL,
4993 4994 4995
			     skb_peek(&sk->sk_receive_queue),
			     NULL,
			     tp->copied_seq, tp->rcv_nxt);
4996
	sk_mem_reclaim(sk);
L
Linus Torvalds 已提交
4997 4998 4999 5000 5001 5002 5003

	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
		return 0;

	/* Collapsing did not help, destructive actions follow.
	 * This must not ever occur. */

5004
	tcp_prune_ofo_queue(sk);
L
Linus Torvalds 已提交
5005 5006 5007 5008 5009 5010 5011 5012

	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
		return 0;

	/* If we are really being abused, tell the caller to silently
	 * drop receive data on the floor.  It will get retransmitted
	 * and hopefully then we'll have sufficient space.
	 */
5013
	NET_INC_STATS(sock_net(sk), LINUX_MIB_RCVPRUNED);
L
Linus Torvalds 已提交
5014 5015

	/* Massive buffer overcommit. */
5016
	tp->pred_flags = 0;
L
Linus Torvalds 已提交
5017 5018 5019
	return -1;
}

E
Eric Dumazet 已提交
5020
static bool tcp_should_expand_sndbuf(const struct sock *sk)
5021
{
5022
	const struct tcp_sock *tp = tcp_sk(sk);
5023

5024 5025 5026 5027
	/* If the user specified a specific send buffer setting, do
	 * not modify it.
	 */
	if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
E
Eric Dumazet 已提交
5028
		return false;
5029 5030

	/* If we are under global TCP memory pressure, do not expand.  */
5031
	if (tcp_under_memory_pressure(sk))
E
Eric Dumazet 已提交
5032
		return false;
5033 5034

	/* If we are under soft global TCP memory pressure, do not expand.  */
5035
	if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, 0))
E
Eric Dumazet 已提交
5036
		return false;
5037 5038

	/* If we filled the congestion window, do not expand.  */
5039
	if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
E
Eric Dumazet 已提交
5040
		return false;
5041

E
Eric Dumazet 已提交
5042
	return true;
5043
}
L
Linus Torvalds 已提交
5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054

/* When incoming ACK allowed to free some skb from write_queue,
 * we remember this event in flag SOCK_QUEUE_SHRUNK and wake up socket
 * on the exit from tcp input handler.
 *
 * PROBLEM: sndbuf expansion does not work well with largesend.
 */
static void tcp_new_space(struct sock *sk)
{
	struct tcp_sock *tp = tcp_sk(sk);

5055
	if (tcp_should_expand_sndbuf(sk)) {
E
Eric Dumazet 已提交
5056
		tcp_sndbuf_expand(sk);
5057
		tp->snd_cwnd_stamp = tcp_jiffies32;
L
Linus Torvalds 已提交
5058 5059 5060 5061 5062
	}

	sk->sk_write_space(sk);
}

S
Stephen Hemminger 已提交
5063
static void tcp_check_space(struct sock *sk)
L
Linus Torvalds 已提交
5064 5065 5066
{
	if (sock_flag(sk, SOCK_QUEUE_SHRUNK)) {
		sock_reset_flag(sk, SOCK_QUEUE_SHRUNK);
5067
		/* pairs with tcp_poll() */
5068
		smp_mb();
L
Linus Torvalds 已提交
5069
		if (sk->sk_socket &&
5070
		    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
L
Linus Torvalds 已提交
5071
			tcp_new_space(sk);
5072 5073 5074
			if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
				tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
		}
L
Linus Torvalds 已提交
5075 5076 5077
	}
}

5078
static inline void tcp_data_snd_check(struct sock *sk)
L
Linus Torvalds 已提交
5079
{
5080
	tcp_push_pending_frames(sk);
L
Linus Torvalds 已提交
5081 5082 5083 5084 5085 5086 5087 5088 5089
	tcp_check_space(sk);
}

/*
 * Check if sending an ack is needed.
 */
static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
{
	struct tcp_sock *tp = tcp_sk(sk);
E
Eric Dumazet 已提交
5090
	unsigned long rtt, delay;
L
Linus Torvalds 已提交
5091 5092

	    /* More than one full frame received... */
5093
	if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&
L
Linus Torvalds 已提交
5094
	     /* ... and right edge of window advances far enough.
5095 5096 5097
	      * (tcp_recvmsg() will send ACK otherwise).
	      * If application uses SO_RCVLOWAT, we want send ack now if
	      * we have not received enough bytes to satisfy the condition.
L
Linus Torvalds 已提交
5098
	      */
5099 5100
	    (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||
	     __tcp_select_window(sk) >= tp->rcv_wnd)) ||
L
Linus Torvalds 已提交
5101
	    /* We ACK each frame or... */
E
Eric Dumazet 已提交
5102 5103
	    tcp_in_quickack_mode(sk)) {
send_now:
L
Linus Torvalds 已提交
5104
		tcp_send_ack(sk);
E
Eric Dumazet 已提交
5105 5106 5107 5108
		return;
	}

	if (!ofo_possible || RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
L
Linus Torvalds 已提交
5109
		tcp_send_delayed_ack(sk);
E
Eric Dumazet 已提交
5110
		return;
L
Linus Torvalds 已提交
5111
	}
E
Eric Dumazet 已提交
5112

E
Eric Dumazet 已提交
5113 5114
	if (!tcp_is_sack(tp) ||
	    tp->compressed_ack >= sock_net(sk)->ipv4.sysctl_tcp_comp_sack_nr)
E
Eric Dumazet 已提交
5115 5116 5117 5118 5119 5120
		goto send_now;
	tp->compressed_ack++;

	if (hrtimer_is_queued(&tp->compressed_ack_timer))
		return;

5121
	/* compress ack timer : 5 % of rtt, but no more than tcp_comp_sack_delay_ns */
E
Eric Dumazet 已提交
5122 5123 5124 5125 5126

	rtt = tp->rcv_rtt_est.rtt_us;
	if (tp->srtt_us && tp->srtt_us < rtt)
		rtt = tp->srtt_us;

5127
	delay = min_t(unsigned long, sock_net(sk)->ipv4.sysctl_tcp_comp_sack_delay_ns,
E
Eric Dumazet 已提交
5128 5129 5130 5131
		      rtt * (NSEC_PER_USEC >> 3)/20);
	sock_hold(sk);
	hrtimer_start(&tp->compressed_ack_timer, ns_to_ktime(delay),
		      HRTIMER_MODE_REL_PINNED_SOFT);
L
Linus Torvalds 已提交
5132 5133
}

S
Stephen Hemminger 已提交
5134
static inline void tcp_ack_snd_check(struct sock *sk)
L
Linus Torvalds 已提交
5135
{
5136
	if (!inet_csk_ack_scheduled(sk)) {
L
Linus Torvalds 已提交
5137 5138 5139 5140 5141 5142 5143 5144
		/* We sent a data segment already. */
		return;
	}
	__tcp_ack_snd_check(sk, 1);
}

/*
 *	This routine is only called when we have urgent data
S
Stephen Hemminger 已提交
5145
 *	signaled. Its the 'slow' part of tcp_urg. It could be
L
Linus Torvalds 已提交
5146 5147 5148 5149 5150 5151
 *	moved inline now as tcp_urg is only called from one
 *	place. We handle URGent data wrong. We have to - as
 *	BSD still doesn't use the correction from RFC961.
 *	For 1003.1g we should support a new option TCP_STDURG to permit
 *	either form (or just set the sysctl tcp_stdurg).
 */
5152

5153
static void tcp_check_urg(struct sock *sk, const struct tcphdr *th)
L
Linus Torvalds 已提交
5154 5155 5156 5157
{
	struct tcp_sock *tp = tcp_sk(sk);
	u32 ptr = ntohs(th->urg_ptr);

5158
	if (ptr && !sock_net(sk)->ipv4.sysctl_tcp_stdurg)
L
Linus Torvalds 已提交
5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189
		ptr--;
	ptr += ntohl(th->seq);

	/* Ignore urgent data that we've already seen and read. */
	if (after(tp->copied_seq, ptr))
		return;

	/* Do not replay urg ptr.
	 *
	 * NOTE: interesting situation not covered by specs.
	 * Misbehaving sender may send urg ptr, pointing to segment,
	 * which we already have in ofo queue. We are not able to fetch
	 * such data and will stay in TCP_URG_NOTYET until will be eaten
	 * by recvmsg(). Seems, we are not obliged to handle such wicked
	 * situations. But it is worth to think about possibility of some
	 * DoSes using some hypothetical application level deadlock.
	 */
	if (before(ptr, tp->rcv_nxt))
		return;

	/* Do we already have a newer (or duplicate) urgent pointer? */
	if (tp->urg_data && !after(ptr, tp->urg_seq))
		return;

	/* Tell the world about our new urgent pointer. */
	sk_send_sigurg(sk);

	/* We may be adding urgent data when the last byte read was
	 * urgent. To do this requires some care. We cannot just ignore
	 * tp->copied_seq since we would read the last urgent byte again
	 * as data, nor can we alter copied_seq until this data arrives
S
Stephen Hemminger 已提交
5190
	 * or we break the semantics of SIOCATMARK (and thus sockatmark())
L
Linus Torvalds 已提交
5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201
	 *
	 * NOTE. Double Dutch. Rendering to plain English: author of comment
	 * above did something sort of 	send("A", MSG_OOB); send("B", MSG_OOB);
	 * and expect that both A and B disappear from stream. This is _wrong_.
	 * Though this happens in BSD with high probability, this is occasional.
	 * Any application relying on this is buggy. Note also, that fix "works"
	 * only in this artificial test. Insert some normal data between A and B and we will
	 * decline of BSD again. Verdict: it is better to remove to trap
	 * buggy users.
	 */
	if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
5202
	    !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
L
Linus Torvalds 已提交
5203 5204 5205
		struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
		tp->copied_seq++;
		if (skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) {
D
David S. Miller 已提交
5206
			__skb_unlink(skb, &sk->sk_receive_queue);
L
Linus Torvalds 已提交
5207 5208 5209 5210
			__kfree_skb(skb);
		}
	}

5211 5212
	tp->urg_data = TCP_URG_NOTYET;
	tp->urg_seq = ptr;
5213 5214 5215

	/* Disable header prediction. */
	tp->pred_flags = 0;
L
Linus Torvalds 已提交
5216 5217 5218
}

/* This is the 'fast' part of urgent handling. */
5219
static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th)
L
Linus Torvalds 已提交
5220 5221 5222 5223 5224
{
	struct tcp_sock *tp = tcp_sk(sk);

	/* Check if we get a new urgent pointer - normally not. */
	if (th->urg)
5225
		tcp_check_urg(sk, th);
L
Linus Torvalds 已提交
5226 5227 5228 5229 5230 5231

	/* Do we wait for any urgent data? - normally not... */
	if (tp->urg_data == TCP_URG_NOTYET) {
		u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) -
			  th->syn;

5232
		/* Is the urgent pointer pointing into this packet? */
L
Linus Torvalds 已提交
5233 5234 5235 5236 5237 5238
		if (ptr < skb->len) {
			u8 tmp;
			if (skb_copy_bits(skb, ptr, &tmp, 1))
				BUG();
			tp->urg_data = TCP_URG_VALID | tmp;
			if (!sock_flag(sk, SOCK_DEAD))
5239
				sk->sk_data_ready(sk);
L
Linus Torvalds 已提交
5240 5241 5242 5243
		}
	}
}

5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260
/* Accept RST for rcv_nxt - 1 after a FIN.
 * When tcp connections are abruptly terminated from Mac OSX (via ^C), a
 * FIN is sent followed by a RST packet. The RST is sent with the same
 * sequence number as the FIN, and thus according to RFC 5961 a challenge
 * ACK should be sent. However, Mac OSX rate limits replies to challenge
 * ACKs on the closed socket. In addition middleboxes can drop either the
 * challenge ACK or a subsequent RST.
 */
static bool tcp_reset_check(const struct sock *sk, const struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);

	return unlikely(TCP_SKB_CB(skb)->seq == (tp->rcv_nxt - 1) &&
			(1 << sk->sk_state) & (TCPF_CLOSE_WAIT | TCPF_LAST_ACK |
					       TCPF_CLOSING));
}

5261 5262 5263
/* Does PAWS and seqno based validation of an incoming segment, flags will
 * play significant role here.
 */
E
Eric Dumazet 已提交
5264 5265
static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
				  const struct tcphdr *th, int syn_inerr)
5266 5267
{
	struct tcp_sock *tp = tcp_sk(sk);
5268
	bool rst_seq_match = false;
5269 5270

	/* RFC1323: H1. Apply PAWS check first. */
5271 5272
	if (tcp_fast_parse_options(sock_net(sk), skb, th, tp) &&
	    tp->rx_opt.saw_tstamp &&
5273 5274
	    tcp_paws_discard(sk, skb)) {
		if (!th->rst) {
5275
			NET_INC_STATS(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
5276 5277 5278 5279
			if (!tcp_oow_rate_limited(sock_net(sk), skb,
						  LINUX_MIB_TCPACKSKIPPEDPAWS,
						  &tp->last_oow_ack_time))
				tcp_send_dupack(sk, skb);
5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292
			goto discard;
		}
		/* Reset is accepted even if it did not pass PAWS. */
	}

	/* Step 1: check sequence number */
	if (!tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)) {
		/* RFC793, page 37: "In all states except SYN-SENT, all reset
		 * (RST) segments are validated by checking their SEQ-fields."
		 * And page 69: "If an incoming segment is not acceptable,
		 * an acknowledgment should be sent in reply (unless the RST
		 * bit is set, if so drop the segment and return)".
		 */
5293 5294 5295
		if (!th->rst) {
			if (th->syn)
				goto syn_challenge;
5296 5297 5298 5299
			if (!tcp_oow_rate_limited(sock_net(sk), skb,
						  LINUX_MIB_TCPACKSKIPPEDSEQ,
						  &tp->last_oow_ack_time))
				tcp_send_dupack(sk, skb);
5300 5301
		} else if (tcp_reset_check(sk, skb)) {
			tcp_reset(sk);
5302
		}
5303 5304 5305 5306 5307
		goto discard;
	}

	/* Step 2: check RST bit */
	if (th->rst) {
5308 5309 5310 5311
		/* RFC 5961 3.2 (extend to match against (RCV.NXT - 1) after a
		 * FIN and SACK too if available):
		 * If seq num matches RCV.NXT or (RCV.NXT - 1) after a FIN, or
		 * the right-most SACK block,
5312
		 * then
E
Eric Dumazet 已提交
5313 5314 5315 5316
		 *     RESET the connection
		 * else
		 *     Send a challenge ACK
		 */
5317 5318
		if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt ||
		    tcp_reset_check(sk, skb)) {
5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336
			rst_seq_match = true;
		} else if (tcp_is_sack(tp) && tp->rx_opt.num_sacks > 0) {
			struct tcp_sack_block *sp = &tp->selective_acks[0];
			int max_sack = sp[0].end_seq;
			int this_sack;

			for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;
			     ++this_sack) {
				max_sack = after(sp[this_sack].end_seq,
						 max_sack) ?
					sp[this_sack].end_seq : max_sack;
			}

			if (TCP_SKB_CB(skb)->seq == max_sack)
				rst_seq_match = true;
		}

		if (rst_seq_match)
E
Eric Dumazet 已提交
5337
			tcp_reset(sk);
5338 5339 5340 5341 5342 5343 5344
		else {
			/* Disable TFO if RST is out-of-order
			 * and no data has been received
			 * for current active TFO socket
			 */
			if (tp->syn_fastopen && !tp->data_segs_in &&
			    sk->sk_state == TCP_ESTABLISHED)
5345
				tcp_fastopen_active_disable(sk);
5346
			tcp_send_challenge_ack(sk, skb);
5347
		}
5348 5349 5350 5351 5352
		goto discard;
	}

	/* step 3: check security and precedence [ignored] */

E
Eric Dumazet 已提交
5353
	/* step 4: Check for a SYN
5354
	 * RFC 5961 4.2 : Send a challenge ack
E
Eric Dumazet 已提交
5355 5356
	 */
	if (th->syn) {
5357
syn_challenge:
5358
		if (syn_inerr)
5359 5360
			TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE);
5361
		tcp_send_challenge_ack(sk, skb);
E
Eric Dumazet 已提交
5362
		goto discard;
5363 5364
	}

E
Eric Dumazet 已提交
5365
	return true;
5366 5367

discard:
5368
	tcp_drop(sk, skb);
E
Eric Dumazet 已提交
5369
	return false;
5370 5371
}

L
Linus Torvalds 已提交
5372
/*
5373
 *	TCP receive function for the ESTABLISHED state.
5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393
 *
 *	It is split into a fast path and a slow path. The fast path is
 * 	disabled when:
 *	- A zero window was announced from us - zero window probing
 *        is only handled properly in the slow path.
 *	- Out of order segments arrived.
 *	- Urgent data is expected.
 *	- There is no buffer space left
 *	- Unexpected TCP flags/window values/header lengths are received
 *	  (detected by checking the TCP header against pred_flags)
 *	- Data is sent in both directions. Fast path only supports pure senders
 *	  or pure receivers (this means either the sequence number or the ack
 *	  value must stay constant)
 *	- Unexpected TCP option.
 *
 *	When these conditions are not satisfied it drops into a standard
 *	receive procedure patterned after RFC793 to handle all cases.
 *	The first three cases are guaranteed by proper pred_flags setting,
 *	the rest is checked inline. Fast processing is turned on in
 *	tcp_data_queue when everything is OK.
L
Linus Torvalds 已提交
5394
 */
5395
void tcp_rcv_established(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
5396
{
5397
	const struct tcphdr *th = (const struct tcphdr *)skb->data;
L
Linus Torvalds 已提交
5398
	struct tcp_sock *tp = tcp_sk(sk);
5399
	unsigned int len = skb->len;
L
Linus Torvalds 已提交
5400

5401 5402 5403
	/* TCP congestion window tracking */
	trace_tcp_probe(sk, skb);

5404
	tcp_mstamp_refresh(tp);
5405
	if (unlikely(!sk->sk_rx_dst))
E
Eric Dumazet 已提交
5406
		inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420
	/*
	 *	Header prediction.
	 *	The code loosely follows the one in the famous
	 *	"30 instruction TCP receive" Van Jacobson mail.
	 *
	 *	Van's trick is to deposit buffers into socket queue
	 *	on a device interrupt, to call tcp_recv function
	 *	on the receive process context and checksum and copy
	 *	the buffer to user space. smart...
	 *
	 *	Our current scheme is not silly either but we take the
	 *	extra cost of the net_bh soft interrupt processing...
	 *	We do checksum and copy also but from device to kernel.
	 */
L
Linus Torvalds 已提交
5421 5422 5423

	tp->rx_opt.saw_tstamp = 0;

5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523
	/*	pred_flags is 0xS?10 << 16 + snd_wnd
	 *	if header_prediction is to be made
	 *	'S' will always be tp->tcp_header_len >> 2
	 *	'?' will be 0 for the fast path, otherwise pred_flags is 0 to
	 *  turn it off	(when there are holes in the receive
	 *	 space for instance)
	 *	PSH flag is ignored.
	 */

	if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags &&
	    TCP_SKB_CB(skb)->seq == tp->rcv_nxt &&
	    !after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
		int tcp_header_len = tp->tcp_header_len;

		/* Timestamp header prediction: tcp_header_len
		 * is automatically equal to th->doff*4 due to pred_flags
		 * match.
		 */

		/* Check timestamp */
		if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) {
			/* No? Slow path! */
			if (!tcp_parse_aligned_timestamp(tp, th))
				goto slow_path;

			/* If PAWS failed, check it more carefully in slow path */
			if ((s32)(tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent) < 0)
				goto slow_path;

			/* DO NOT update ts_recent here, if checksum fails
			 * and timestamp was corrupted part, it will result
			 * in a hung connection since we will drop all
			 * future packets due to the PAWS test.
			 */
		}

		if (len <= tcp_header_len) {
			/* Bulk data transfer: sender */
			if (len == tcp_header_len) {
				/* Predicted packet is in window by definition.
				 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
				 * Hence, check seq<=rcv_wup reduces to:
				 */
				if (tcp_header_len ==
				    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
				    tp->rcv_nxt == tp->rcv_wup)
					tcp_store_ts_recent(tp);

				/* We know that such packets are checksummed
				 * on entry.
				 */
				tcp_ack(sk, skb, 0);
				__kfree_skb(skb);
				tcp_data_snd_check(sk);
				return;
			} else { /* Header too small */
				TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
				goto discard;
			}
		} else {
			int eaten = 0;
			bool fragstolen = false;

			if (tcp_checksum_complete(skb))
				goto csum_error;

			if ((int)skb->truesize > sk->sk_forward_alloc)
				goto step5;

			/* Predicted packet is in window by definition.
			 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
			 * Hence, check seq<=rcv_wup reduces to:
			 */
			if (tcp_header_len ==
			    (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
			    tp->rcv_nxt == tp->rcv_wup)
				tcp_store_ts_recent(tp);

			tcp_rcv_rtt_measure_ts(sk, skb);

			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPHITS);

			/* Bulk data transfer: receiver */
			eaten = tcp_queue_rcv(sk, skb, tcp_header_len,
					      &fragstolen);

			tcp_event_data_recv(sk, skb);

			if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) {
				/* Well, only one small jumplet in fast path... */
				tcp_ack(sk, skb, FLAG_DATA);
				tcp_data_snd_check(sk);
				if (!inet_csk_ack_scheduled(sk))
					goto no_ack;
			}

			__tcp_ack_snd_check(sk, 0);
no_ack:
			if (eaten)
				kfree_skb_partial(skb, fragstolen);
5524
			tcp_data_ready(sk);
5525 5526 5527 5528 5529
			return;
		}
	}

slow_path:
5530
	if (len < (th->doff << 2) || tcp_checksum_complete(skb))
L
Linus Torvalds 已提交
5531 5532
		goto csum_error;

5533
	if (!th->ack && !th->rst && !th->syn)
5534 5535
		goto discard;

5536 5537 5538 5539
	/*
	 *	Standard slow path.
	 */

E
Eric Dumazet 已提交
5540
	if (!tcp_validate_incoming(sk, skb, th, 1))
5541
		return;
L
Linus Torvalds 已提交
5542

5543 5544
step5:
	if (tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT) < 0)
5545
		goto discard;
L
Linus Torvalds 已提交
5546

5547
	tcp_rcv_rtt_measure_ts(sk, skb);
L
Linus Torvalds 已提交
5548 5549 5550 5551 5552 5553 5554

	/* Process urgent data. */
	tcp_urg(sk, skb, th);

	/* step 7: process the segment text */
	tcp_data_queue(sk, skb);

5555
	tcp_data_snd_check(sk);
L
Linus Torvalds 已提交
5556
	tcp_ack_snd_check(sk);
5557
	return;
L
Linus Torvalds 已提交
5558 5559

csum_error:
5560 5561
	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
L
Linus Torvalds 已提交
5562 5563

discard:
5564
	tcp_drop(sk, skb);
L
Linus Torvalds 已提交
5565
}
E
Eric Dumazet 已提交
5566
EXPORT_SYMBOL(tcp_rcv_established);
L
Linus Torvalds 已提交
5567

P
Pavel Emelyanov 已提交
5568 5569 5570 5571 5572 5573
void tcp_finish_connect(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);
	struct inet_connection_sock *icsk = inet_csk(sk);

	tcp_set_state(sk, TCP_ESTABLISHED);
5574
	icsk->icsk_ack.lrcvtime = tcp_jiffies32;
P
Pavel Emelyanov 已提交
5575

5576
	if (skb) {
E
Eric Dumazet 已提交
5577
		icsk->icsk_af_ops->sk_rx_dst_set(sk, skb);
P
Pavel Emelyanov 已提交
5578
		security_inet_conn_established(sk, skb);
D
David S. Miller 已提交
5579
	}
P
Pavel Emelyanov 已提交
5580

5581
	tcp_init_transfer(sk, BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB);
P
Pavel Emelyanov 已提交
5582 5583 5584 5585

	/* Prevent spurious tcp_cwnd_restart() on first data
	 * packet.
	 */
5586
	tp->lsndtime = tcp_jiffies32;
P
Pavel Emelyanov 已提交
5587 5588 5589

	if (sock_flag(sk, SOCK_KEEPOPEN))
		inet_csk_reset_keepalive_timer(sk, keepalive_time_when(tp));
5590 5591 5592 5593 5594

	if (!tp->rx_opt.snd_wscale)
		__tcp_fast_path_on(tp, tp->snd_wnd);
	else
		tp->pred_flags = 0;
P
Pavel Emelyanov 已提交
5595 5596
}

5597 5598 5599 5600
static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack,
				    struct tcp_fastopen_cookie *cookie)
{
	struct tcp_sock *tp = tcp_sk(sk);
5601
	struct sk_buff *data = tp->syn_data ? tcp_rtx_queue_head(sk) : NULL;
5602 5603
	u16 mss = tp->rx_opt.mss_clamp, try_exp = 0;
	bool syn_drop = false;
5604 5605 5606 5607 5608 5609 5610

	if (mss == tp->rx_opt.user_mss) {
		struct tcp_options_received opt;

		/* Get original SYNACK MSS value if user MSS sets mss_clamp */
		tcp_clear_options(&opt);
		opt.user_mss = opt.mss_clamp = 0;
5611
		tcp_parse_options(sock_net(sk), synack, &opt, 0, NULL);
5612 5613 5614
		mss = opt.mss_clamp;
	}

5615 5616
	if (!tp->syn_fastopen) {
		/* Ignore an unsolicited cookie */
5617
		cookie->len = -1;
5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631
	} else if (tp->total_retrans) {
		/* SYN timed out and the SYN-ACK neither has a cookie nor
		 * acknowledges data. Presumably the remote received only
		 * the retransmitted (regular) SYNs: either the original
		 * SYN-data or the corresponding SYN-ACK was dropped.
		 */
		syn_drop = (cookie->len < 0 && data);
	} else if (cookie->len < 0 && !tp->syn_data) {
		/* We requested a cookie but didn't get it. If we did not use
		 * the (old) exp opt format then try so next time (try_exp=1).
		 * Otherwise we go back to use the RFC7413 opt (try_exp=2).
		 */
		try_exp = tp->syn_fastopen_exp ? 2 : 1;
	}
5632

5633
	tcp_fastopen_cache_set(sk, mss, cookie, syn_drop, try_exp);
5634 5635

	if (data) { /* Retransmit unacked data in SYN */
5636 5637
		skb_rbtree_walk_from(data) {
			if (__tcp_retransmit_skb(sk, data, 1))
5638 5639
				break;
		}
5640
		tcp_rearm_rto(sk);
5641
		NET_INC_STATS(sock_net(sk),
5642
				LINUX_MIB_TCPFASTOPENACTIVEFAIL);
5643 5644
		return true;
	}
Y
Yuchung Cheng 已提交
5645
	tp->syn_data_acked = tp->syn_data;
5646 5647 5648 5649 5650 5651
	if (tp->syn_data_acked) {
		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
		/* SYN-data is counted as two separate packets in tcp_ack() */
		if (tp->delivered > 1)
			--tp->delivered;
	}
5652 5653 5654

	tcp_fastopen_add_skb(sk, synack);

5655 5656 5657
	return false;
}

5658 5659 5660 5661 5662 5663 5664 5665 5666 5667
static void smc_check_reset_syn(struct tcp_sock *tp)
{
#if IS_ENABLED(CONFIG_SMC)
	if (static_branch_unlikely(&tcp_have_smc)) {
		if (tp->syn_smc && !tp->rx_opt.smc_ok)
			tp->syn_smc = 0;
	}
#endif
}

L
Linus Torvalds 已提交
5668
static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
5669
					 const struct tcphdr *th)
L
Linus Torvalds 已提交
5670
{
5671
	struct inet_connection_sock *icsk = inet_csk(sk);
5672
	struct tcp_sock *tp = tcp_sk(sk);
5673
	struct tcp_fastopen_cookie foc = { .len = -1 };
5674
	int saved_clamp = tp->rx_opt.mss_clamp;
5675
	bool fastopen_fail;
L
Linus Torvalds 已提交
5676

5677
	tcp_parse_options(sock_net(sk), skb, &tp->rx_opt, 0, &foc);
5678
	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
5679
		tp->rx_opt.rcv_tsecr -= tp->tsoffset;
L
Linus Torvalds 已提交
5680 5681 5682 5683 5684 5685 5686 5687 5688 5689

	if (th->ack) {
		/* rfc793:
		 * "If the state is SYN-SENT then
		 *    first check the ACK bit
		 *      If the ACK bit is set
		 *	  If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send
		 *        a reset (unless the RST bit is set, if so drop
		 *        the segment and return)"
		 */
5690 5691
		if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) ||
		    after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt))
L
Linus Torvalds 已提交
5692 5693 5694 5695
			goto reset_and_undo;

		if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
		    !between(tp->rx_opt.rcv_tsecr, tp->retrans_stamp,
5696
			     tcp_time_stamp(tp))) {
5697
			NET_INC_STATS(sock_net(sk),
5698
					LINUX_MIB_PAWSACTIVEREJECTED);
L
Linus Torvalds 已提交
5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731
			goto reset_and_undo;
		}

		/* Now ACK is acceptable.
		 *
		 * "If the RST bit is set
		 *    If the ACK was acceptable then signal the user "error:
		 *    connection reset", drop the segment, enter CLOSED state,
		 *    delete TCB, and return."
		 */

		if (th->rst) {
			tcp_reset(sk);
			goto discard;
		}

		/* rfc793:
		 *   "fifth, if neither of the SYN or RST bits is set then
		 *    drop the segment and return."
		 *
		 *    See note below!
		 *                                        --ANK(990513)
		 */
		if (!th->syn)
			goto discard_and_undo;

		/* rfc793:
		 *   "If the SYN bit is on ...
		 *    are acceptable then ...
		 *    (our SYN has been ACKed), change the connection
		 *    state to ESTABLISHED..."
		 */

5732
		tcp_ecn_rcv_synack(tp, th);
L
Linus Torvalds 已提交
5733

5734
		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
5735
		tcp_ack(sk, skb, FLAG_SLOWPATH);
L
Linus Torvalds 已提交
5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762

		/* Ok.. it's good. Set up sequence numbers and
		 * move to established.
		 */
		tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;

		/* RFC1323: The window in SYN & SYN/ACK segments is
		 * never scaled.
		 */
		tp->snd_wnd = ntohs(th->window);

		if (!tp->rx_opt.wscale_ok) {
			tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0;
			tp->window_clamp = min(tp->window_clamp, 65535U);
		}

		if (tp->rx_opt.saw_tstamp) {
			tp->rx_opt.tstamp_ok	   = 1;
			tp->tcp_header_len =
				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
			tp->advmss	    -= TCPOLEN_TSTAMP_ALIGNED;
			tcp_store_ts_recent(tp);
		} else {
			tp->tcp_header_len = sizeof(struct tcphdr);
		}

5763
		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
L
Linus Torvalds 已提交
5764 5765 5766 5767 5768 5769
		tcp_initialize_rcv_mss(sk);

		/* Remember, tcp_poll() does not lock socket!
		 * Change state from SYN-SENT only after copied_seq
		 * is initialized. */
		tp->copied_seq = tp->rcv_nxt;
5770

5771 5772
		smc_check_reset_syn(tp);

R
Ralf Baechle 已提交
5773
		smp_mb();
L
Linus Torvalds 已提交
5774

P
Pavel Emelyanov 已提交
5775
		tcp_finish_connect(sk, skb);
L
Linus Torvalds 已提交
5776

5777 5778
		fastopen_fail = (tp->syn_fastopen || tp->syn_data) &&
				tcp_rcv_fastopen_synack(sk, skb, &foc);
5779

5780 5781 5782 5783 5784 5785
		if (!sock_flag(sk, SOCK_DEAD)) {
			sk->sk_state_change(sk);
			sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
		}
		if (fastopen_fail)
			return -1;
5786 5787 5788
		if (sk->sk_write_pending ||
		    icsk->icsk_accept_queue.rskq_defer_accept ||
		    icsk->icsk_ack.pingpong) {
L
Linus Torvalds 已提交
5789 5790 5791 5792 5793 5794 5795
			/* Save one ACK. Data will be ready after
			 * several ticks, if write_pending is set.
			 *
			 * It may be deleted, but with this feature tcpdumps
			 * look so _wonderfully_ clever, that I was not able
			 * to stand against the temptation 8)     --ANK
			 */
5796
			inet_csk_schedule_ack(sk);
5797
			tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
5798 5799
			inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
						  TCP_DELACK_MAX, TCP_RTO_MAX);
L
Linus Torvalds 已提交
5800 5801

discard:
5802
			tcp_drop(sk, skb);
L
Linus Torvalds 已提交
5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822
			return 0;
		} else {
			tcp_send_ack(sk);
		}
		return -1;
	}

	/* No ACK in the segment */

	if (th->rst) {
		/* rfc793:
		 * "If the RST bit is set
		 *
		 *      Otherwise (no ACK) drop the segment and return."
		 */

		goto discard_and_undo;
	}

	/* PAWS check. */
5823
	if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp &&
I
Ilpo Järvinen 已提交
5824
	    tcp_paws_reject(&tp->rx_opt, 0))
L
Linus Torvalds 已提交
5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843
		goto discard_and_undo;

	if (th->syn) {
		/* We see SYN without ACK. It is attempt of
		 * simultaneous connect with crossed SYNs.
		 * Particularly, it can be connect to self.
		 */
		tcp_set_state(sk, TCP_SYN_RECV);

		if (tp->rx_opt.saw_tstamp) {
			tp->rx_opt.tstamp_ok = 1;
			tcp_store_ts_recent(tp);
			tp->tcp_header_len =
				sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
		} else {
			tp->tcp_header_len = sizeof(struct tcphdr);
		}

		tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
5844
		tp->copied_seq = tp->rcv_nxt;
L
Linus Torvalds 已提交
5845 5846 5847 5848 5849 5850 5851 5852 5853
		tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;

		/* RFC1323: The window in SYN & SYN/ACK segments is
		 * never scaled.
		 */
		tp->snd_wnd    = ntohs(th->window);
		tp->snd_wl1    = TCP_SKB_CB(skb)->seq;
		tp->max_window = tp->snd_wnd;

5854
		tcp_ecn_rcv_syn(tp, th);
L
Linus Torvalds 已提交
5855

J
John Heffner 已提交
5856
		tcp_mtup_init(sk);
5857
		tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
L
Linus Torvalds 已提交
5858 5859 5860 5861 5862
		tcp_initialize_rcv_mss(sk);

		tcp_send_synack(sk);
#if 0
		/* Note, we could accept data and URG from this segment.
5863 5864 5865
		 * There are no obstacles to make this (except that we must
		 * either change tcp_recvmsg() to prevent it from returning data
		 * before 3WHS completes per RFC793, or employ TCP Fast Open).
L
Linus Torvalds 已提交
5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894
		 *
		 * However, if we ignore data in ACKless segments sometimes,
		 * we have no reasons to accept it sometimes.
		 * Also, seems the code doing it in step6 of tcp_rcv_state_process
		 * is not flawless. So, discard packet for sanity.
		 * Uncomment this return to process the data.
		 */
		return -1;
#else
		goto discard;
#endif
	}
	/* "fifth, if neither of the SYN or RST bits is set then
	 * drop the segment and return."
	 */

discard_and_undo:
	tcp_clear_options(&tp->rx_opt);
	tp->rx_opt.mss_clamp = saved_clamp;
	goto discard;

reset_and_undo:
	tcp_clear_options(&tp->rx_opt);
	tp->rx_opt.mss_clamp = saved_clamp;
	return 1;
}

/*
 *	This function implements the receiving procedure of RFC 793 for
5895
 *	all states except ESTABLISHED and TIME_WAIT.
L
Linus Torvalds 已提交
5896 5897 5898
 *	It's called from both tcp_v4_rcv and tcp_v6_rcv and should be
 *	address independent.
 */
5899

5900
int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
5901 5902
{
	struct tcp_sock *tp = tcp_sk(sk);
5903
	struct inet_connection_sock *icsk = inet_csk(sk);
5904
	const struct tcphdr *th = tcp_hdr(skb);
5905
	struct request_sock *req;
L
Linus Torvalds 已提交
5906
	int queued = 0;
5907
	bool acceptable;
L
Linus Torvalds 已提交
5908 5909 5910 5911 5912 5913

	switch (sk->sk_state) {
	case TCP_CLOSE:
		goto discard;

	case TCP_LISTEN:
S
Stephen Hemminger 已提交
5914
		if (th->ack)
L
Linus Torvalds 已提交
5915 5916
			return 1;

S
Stephen Hemminger 已提交
5917
		if (th->rst)
L
Linus Torvalds 已提交
5918 5919
			goto discard;

S
Stephen Hemminger 已提交
5920
		if (th->syn) {
E
Eric Dumazet 已提交
5921 5922
			if (th->fin)
				goto discard;
5923 5924 5925 5926 5927 5928
			/* It is possible that we process SYN packets from backlog,
			 * so we need to make sure to disable BH right there.
			 */
			local_bh_disable();
			acceptable = icsk->icsk_af_ops->conn_request(sk, skb) >= 0;
			local_bh_enable();
L
Linus Torvalds 已提交
5929

5930 5931
			if (!acceptable)
				return 1;
5932
			consume_skb(skb);
5933
			return 0;
L
Linus Torvalds 已提交
5934 5935 5936 5937
		}
		goto discard;

	case TCP_SYN_SENT:
5938
		tp->rx_opt.saw_tstamp = 0;
5939
		tcp_mstamp_refresh(tp);
5940
		queued = tcp_rcv_synsent_state_process(sk, skb, th);
L
Linus Torvalds 已提交
5941 5942 5943 5944 5945 5946
		if (queued >= 0)
			return queued;

		/* Do step6 onward by hand. */
		tcp_urg(sk, skb, th);
		__kfree_skb(skb);
5947
		tcp_data_snd_check(sk);
L
Linus Torvalds 已提交
5948 5949 5950
		return 0;
	}

5951
	tcp_mstamp_refresh(tp);
5952
	tp->rx_opt.saw_tstamp = 0;
5953
	req = tp->fastopen_rsk;
5954
	if (req) {
5955 5956
		bool req_stolen;

5957
		WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV &&
5958 5959
		    sk->sk_state != TCP_FIN_WAIT1);

5960
		if (!tcp_check_req(sk, skb, req, true, &req_stolen))
5961
			goto discard;
5962
	}
5963

5964
	if (!th->ack && !th->rst && !th->syn)
5965 5966
		goto discard;

5967
	if (!tcp_validate_incoming(sk, skb, th, 0))
E
Eric Dumazet 已提交
5968
		return 0;
L
Linus Torvalds 已提交
5969 5970

	/* step 5: check the ACK field */
5971 5972
	acceptable = tcp_ack(sk, skb, FLAG_SLOWPATH |
				      FLAG_UPDATE_TS_RECENT |
5973
				      FLAG_NO_CHALLENGE_ACK) > 0;
5974

5975 5976 5977 5978 5979 5980
	if (!acceptable) {
		if (sk->sk_state == TCP_SYN_RECV)
			return 1;	/* send one RST */
		tcp_send_challenge_ack(sk, skb);
		goto discard;
	}
5981 5982
	switch (sk->sk_state) {
	case TCP_SYN_RECV:
5983
		tp->delivered++; /* SYN-ACK delivery isn't tracked in tcp_ack */
Y
Yuchung Cheng 已提交
5984 5985 5986
		if (!tp->srtt_us)
			tcp_synack_rtt_meas(sk, req);

5987 5988 5989 5990
		/* Once we leave TCP_SYN_RECV, we no longer need req
		 * so release it.
		 */
		if (req) {
5991
			inet_csk(sk)->icsk_retransmits = 0;
5992
			reqsk_fastopen_remove(sk, req, false);
5993 5994 5995 5996 5997 5998 5999 6000 6001
			/* Re-arm the timer because data may have been sent out.
			 * This is similar to the regular data transmission case
			 * when new data has just been ack'ed.
			 *
			 * (TFO) - we could try to be more aggressive and
			 * retransmitting any data sooner based on when they
			 * are sent out.
			 */
			tcp_rearm_rto(sk);
6002
		} else {
6003
			tcp_init_transfer(sk, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB);
6004 6005 6006 6007 6008
			tp->copied_seq = tp->rcv_nxt;
		}
		smp_mb();
		tcp_set_state(sk, TCP_ESTABLISHED);
		sk->sk_state_change(sk);
L
Linus Torvalds 已提交
6009

6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023
		/* Note, that this wakeup is only for marginal crossed SYN case.
		 * Passively open sockets are not waked up, because
		 * sk->sk_sleep == NULL and sk->sk_socket == NULL.
		 */
		if (sk->sk_socket)
			sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);

		tp->snd_una = TCP_SKB_CB(skb)->ack_seq;
		tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
		tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);

		if (tp->rx_opt.tstamp_ok)
			tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;

6024 6025
		if (!inet_csk(sk)->icsk_ca_ops->cong_control)
			tcp_update_pacing_rate(sk);
6026

6027
		/* Prevent spurious tcp_cwnd_restart() on first data packet */
6028
		tp->lsndtime = tcp_jiffies32;
6029 6030

		tcp_initialize_rcv_mss(sk);
6031
		tcp_fast_path_on(tp);
6032 6033
		break;

6034 6035 6036
	case TCP_FIN_WAIT1: {
		int tmo;

6037 6038 6039 6040 6041
		/* If we enter the TCP_FIN_WAIT1 state and we are a
		 * Fast Open socket and this is the first acceptable
		 * ACK we have received, this would have acknowledged
		 * our SYNACK so stop the SYNACK timer.
		 */
6042
		if (req) {
6043 6044 6045 6046
			/* We no longer need the request sock. */
			reqsk_fastopen_remove(sk, req, false);
			tcp_rearm_rto(sk);
		}
6047 6048
		if (tp->snd_una != tp->write_seq)
			break;
6049

6050 6051
		tcp_set_state(sk, TCP_FIN_WAIT2);
		sk->sk_shutdown |= SEND_SHUTDOWN;
6052

6053
		sk_dst_confirm(sk);
6054

6055 6056 6057 6058 6059
		if (!sock_flag(sk, SOCK_DEAD)) {
			/* Wake up lingering close() */
			sk->sk_state_change(sk);
			break;
		}
L
Linus Torvalds 已提交
6060

6061 6062 6063 6064 6065 6066 6067 6068 6069
		if (tp->linger2 < 0) {
			tcp_done(sk);
			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
			return 1;
		}
		if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
		    after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
			/* Receive out of order FIN after close() */
			if (tp->syn_fastopen && th->fin)
6070
				tcp_fastopen_active_disable(sk);
6071
			tcp_done(sk);
6072
			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089
			return 1;
		}

		tmo = tcp_fin_time(sk);
		if (tmo > TCP_TIMEWAIT_LEN) {
			inet_csk_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN);
		} else if (th->fin || sock_owned_by_user(sk)) {
			/* Bad case. We could lose such FIN otherwise.
			 * It is not a big problem, but it looks confusing
			 * and not so rare event. We still can lose it now,
			 * if it spins in bh_lock_sock(), but it is really
			 * marginal case.
			 */
			inet_csk_reset_keepalive_timer(sk, tmo);
		} else {
			tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
			goto discard;
6090 6091
		}
		break;
6092
	}
L
Linus Torvalds 已提交
6093

6094 6095 6096 6097 6098 6099
	case TCP_CLOSING:
		if (tp->snd_una == tp->write_seq) {
			tcp_time_wait(sk, TCP_TIME_WAIT, 0);
			goto discard;
		}
		break;
L
Linus Torvalds 已提交
6100

6101 6102 6103 6104 6105
	case TCP_LAST_ACK:
		if (tp->snd_una == tp->write_seq) {
			tcp_update_metrics(sk);
			tcp_done(sk);
			goto discard;
L
Linus Torvalds 已提交
6106
		}
6107
		break;
6108
	}
L
Linus Torvalds 已提交
6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119

	/* step 6: check the URG bit */
	tcp_urg(sk, skb, th);

	/* step 7: process the segment text */
	switch (sk->sk_state) {
	case TCP_CLOSE_WAIT:
	case TCP_CLOSING:
	case TCP_LAST_ACK:
		if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
			break;
6120
		/* fall through */
L
Linus Torvalds 已提交
6121 6122 6123
	case TCP_FIN_WAIT1:
	case TCP_FIN_WAIT2:
		/* RFC 793 says to queue data in these states,
6124
		 * RFC 1122 says we MUST send a reset.
L
Linus Torvalds 已提交
6125 6126 6127 6128 6129
		 * BSD 4.4 also does reset.
		 */
		if (sk->sk_shutdown & RCV_SHUTDOWN) {
			if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
			    after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
6130
				NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
L
Linus Torvalds 已提交
6131 6132 6133 6134 6135
				tcp_reset(sk);
				return 1;
			}
		}
		/* Fall through */
6136
	case TCP_ESTABLISHED:
L
Linus Torvalds 已提交
6137 6138 6139 6140 6141 6142 6143
		tcp_data_queue(sk, skb);
		queued = 1;
		break;
	}

	/* tcp_data could move socket to TIME-WAIT */
	if (sk->sk_state != TCP_CLOSE) {
6144
		tcp_data_snd_check(sk);
L
Linus Torvalds 已提交
6145 6146 6147
		tcp_ack_snd_check(sk);
	}

6148
	if (!queued) {
L
Linus Torvalds 已提交
6149
discard:
6150
		tcp_drop(sk, skb);
L
Linus Torvalds 已提交
6151 6152 6153 6154
	}
	return 0;
}
EXPORT_SYMBOL(tcp_rcv_state_process);
O
Octavian Purdila 已提交
6155 6156 6157 6158 6159 6160

static inline void pr_drop_req(struct request_sock *req, __u16 port, int family)
{
	struct inet_request_sock *ireq = inet_rsk(req);

	if (family == AF_INET)
6161 6162
		net_dbg_ratelimited("drop open request from %pI4/%u\n",
				    &ireq->ir_rmt_addr, port);
6163 6164
#if IS_ENABLED(CONFIG_IPV6)
	else if (family == AF_INET6)
6165 6166
		net_dbg_ratelimited("drop open request from %pI6/%u\n",
				    &ireq->ir_v6_rmt_addr, port);
6167
#endif
O
Octavian Purdila 已提交
6168 6169
}

6170 6171 6172 6173 6174
/* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set
 *
 * If we receive a SYN packet with these bits set, it means a
 * network is playing bad games with TOS bits. In order to
 * avoid possible false congestion notifications, we disable
S
stephen hemminger 已提交
6175
 * TCP ECN negotiation.
6176 6177
 *
 * Exception: tcp_ca wants ECN. This is required for DCTCP
6178 6179 6180
 * congestion control: Linux DCTCP asserts ECT on all packets,
 * including SYN, which is most optimal solution; however,
 * others, such as FreeBSD do not.
6181 6182 6183
 */
static void tcp_ecn_create_request(struct request_sock *req,
				   const struct sk_buff *skb,
6184 6185
				   const struct sock *listen_sk,
				   const struct dst_entry *dst)
6186 6187 6188 6189
{
	const struct tcphdr *th = tcp_hdr(skb);
	const struct net *net = sock_net(listen_sk);
	bool th_ecn = th->ece && th->cwr;
6190
	bool ect, ecn_ok;
6191
	u32 ecn_ok_dst;
6192 6193 6194 6195 6196

	if (!th_ecn)
		return;

	ect = !INET_ECN_is_not_ect(TCP_SKB_CB(skb)->ip_dsfield);
6197 6198
	ecn_ok_dst = dst_feature(dst, DST_FEATURE_ECN_MASK);
	ecn_ok = net->ipv4.sysctl_tcp_ecn || ecn_ok_dst;
6199

6200
	if ((!ect && ecn_ok) || tcp_ca_needs_ecn(listen_sk) ||
6201 6202
	    (ecn_ok_dst & DST_FEATURE_ECN_CA) ||
	    tcp_bpf_ca_needs_ecn((struct sock *)req))
6203 6204 6205
		inet_rsk(req)->ecn_ok = 1;
}

6206 6207 6208 6209 6210 6211
static void tcp_openreq_init(struct request_sock *req,
			     const struct tcp_options_received *rx_opt,
			     struct sk_buff *skb, const struct sock *sk)
{
	struct inet_request_sock *ireq = inet_rsk(req);

6212
	req->rsk_rcv_wnd = 0;		/* So that tcp_send_synack() knows! */
6213 6214 6215
	req->cookie_ts = 0;
	tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
	tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
6216
	tcp_rsk(req)->snt_synack = tcp_clock_us();
6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228
	tcp_rsk(req)->last_oow_ack_time = 0;
	req->mss = rx_opt->mss_clamp;
	req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0;
	ireq->tstamp_ok = rx_opt->tstamp_ok;
	ireq->sack_ok = rx_opt->sack_ok;
	ireq->snd_wscale = rx_opt->snd_wscale;
	ireq->wscale_ok = rx_opt->wscale_ok;
	ireq->acked = 0;
	ireq->ecn_ok = 0;
	ireq->ir_rmt_port = tcp_hdr(skb)->source;
	ireq->ir_num = ntohs(tcp_hdr(skb)->dest);
	ireq->ir_mark = inet_request_mark(sk, skb);
6229 6230 6231
#if IS_ENABLED(CONFIG_SMC)
	ireq->smc_ok = rx_opt->smc_ok;
#endif
6232 6233
}

E
Eric Dumazet 已提交
6234
struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops,
6235 6236
				      struct sock *sk_listener,
				      bool attach_listener)
E
Eric Dumazet 已提交
6237
{
6238 6239
	struct request_sock *req = reqsk_alloc(ops, sk_listener,
					       attach_listener);
E
Eric Dumazet 已提交
6240 6241 6242 6243

	if (req) {
		struct inet_request_sock *ireq = inet_rsk(req);

E
Eric Dumazet 已提交
6244
		ireq->ireq_opt = NULL;
6245 6246 6247
#if IS_ENABLED(CONFIG_IPV6)
		ireq->pktopts = NULL;
#endif
6248
		atomic64_set(&ireq->ir_cookie, 0);
E
Eric Dumazet 已提交
6249 6250
		ireq->ireq_state = TCP_NEW_SYN_RECV;
		write_pnet(&ireq->ireq_net, sock_net(sk_listener));
6251
		ireq->ireq_family = sk_listener->sk_family;
E
Eric Dumazet 已提交
6252 6253 6254 6255 6256 6257
	}

	return req;
}
EXPORT_SYMBOL(inet_reqsk_alloc);

6258 6259 6260
/*
 * Return true if a syncookie should be sent
 */
6261
static bool tcp_syn_flood_action(const struct sock *sk,
6262 6263 6264
				 const struct sk_buff *skb,
				 const char *proto)
{
6265
	struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
6266 6267
	const char *msg = "Dropping request";
	bool want_cookie = false;
6268
	struct net *net = sock_net(sk);
6269 6270

#ifdef CONFIG_SYN_COOKIES
6271
	if (net->ipv4.sysctl_tcp_syncookies) {
6272 6273
		msg = "Sending cookies";
		want_cookie = true;
6274
		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES);
6275 6276
	} else
#endif
6277
		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP);
6278

6279
	if (!queue->synflood_warned &&
6280
	    net->ipv4.sysctl_tcp_syncookies != 2 &&
6281
	    xchg(&queue->synflood_warned, 1) == 0)
6282 6283
		pr_info("%s: Possible SYN flooding on port %d. %s.  Check SNMP counters.\n",
			proto, ntohs(tcp_hdr(skb)->dest), msg);
6284

6285 6286 6287
	return want_cookie;
}

6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304
static void tcp_reqsk_record_syn(const struct sock *sk,
				 struct request_sock *req,
				 const struct sk_buff *skb)
{
	if (tcp_sk(sk)->save_syn) {
		u32 len = skb_network_header_len(skb) + tcp_hdrlen(skb);
		u32 *copy;

		copy = kmalloc(len + sizeof(u32), GFP_ATOMIC);
		if (copy) {
			copy[0] = len;
			memcpy(&copy[1], skb_network_header(skb), len);
			req->saved_syn = copy;
		}
	}
}

O
Octavian Purdila 已提交
6305 6306 6307 6308
int tcp_conn_request(struct request_sock_ops *rsk_ops,
		     const struct tcp_request_sock_ops *af_ops,
		     struct sock *sk, struct sk_buff *skb)
{
6309 6310
	struct tcp_fastopen_cookie foc = { .len = -1 };
	__u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
O
Octavian Purdila 已提交
6311 6312
	struct tcp_options_received tmp_opt;
	struct tcp_sock *tp = tcp_sk(sk);
6313
	struct net *net = sock_net(sk);
6314 6315 6316
	struct sock *fastopen_sk = NULL;
	struct request_sock *req;
	bool want_cookie = false;
6317
	struct dst_entry *dst;
O
Octavian Purdila 已提交
6318 6319 6320 6321 6322 6323
	struct flowi fl;

	/* TW buckets are converted to open requests without
	 * limitations, they conserve resources and peer is
	 * evidently real one.
	 */
6324
	if ((net->ipv4.sysctl_tcp_syncookies == 2 ||
O
Octavian Purdila 已提交
6325 6326 6327 6328 6329 6330
	     inet_csk_reqsk_queue_is_full(sk)) && !isn) {
		want_cookie = tcp_syn_flood_action(sk, skb, rsk_ops->slab_name);
		if (!want_cookie)
			goto drop;
	}

6331
	if (sk_acceptq_is_full(sk)) {
6332
		NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
O
Octavian Purdila 已提交
6333 6334 6335
		goto drop;
	}

6336
	req = inet_reqsk_alloc(rsk_ops, sk, !want_cookie);
O
Octavian Purdila 已提交
6337 6338 6339 6340
	if (!req)
		goto drop;

	tcp_rsk(req)->af_specific = af_ops;
6341
	tcp_rsk(req)->ts_off = 0;
O
Octavian Purdila 已提交
6342 6343 6344 6345

	tcp_clear_options(&tmp_opt);
	tmp_opt.mss_clamp = af_ops->mss_clamp;
	tmp_opt.user_mss  = tp->rx_opt.user_mss;
6346 6347
	tcp_parse_options(sock_net(sk), skb, &tmp_opt, 0,
			  want_cookie ? NULL : &foc);
O
Octavian Purdila 已提交
6348 6349 6350 6351

	if (want_cookie && !tmp_opt.saw_tstamp)
		tcp_clear_options(&tmp_opt);

6352 6353 6354
	if (IS_ENABLED(CONFIG_SMC) && want_cookie)
		tmp_opt.smc_ok = 0;

O
Octavian Purdila 已提交
6355 6356
	tmp_opt.tstamp_ok = tmp_opt.saw_tstamp;
	tcp_openreq_init(req, &tmp_opt, skb, sk);
6357
	inet_rsk(req)->no_srccheck = inet_sk(sk)->transparent;
O
Octavian Purdila 已提交
6358

6359
	/* Note: tcp_v6_init_req() might override ir_iif for link locals */
6360
	inet_rsk(req)->ir_iif = inet_request_bound_dev_if(sk, skb);
6361

O
Octavian Purdila 已提交
6362 6363 6364 6365 6366
	af_ops->init_req(req, sk, skb);

	if (security_inet_conn_request(sk, skb, req))
		goto drop_and_free;

6367
	if (tmp_opt.tstamp_ok)
6368
		tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb);
6369

6370 6371 6372 6373
	dst = af_ops->route_req(sk, &fl, req);
	if (!dst)
		goto drop_and_free;

6374
	if (!want_cookie && !isn) {
O
Octavian Purdila 已提交
6375
		/* Kill the following clause, if you dislike this way. */
6376 6377 6378 6379
		if (!net->ipv4.sysctl_tcp_syncookies &&
		    (net->ipv4.sysctl_max_syn_backlog - inet_csk_reqsk_queue_len(sk) <
		     (net->ipv4.sysctl_max_syn_backlog >> 2)) &&
		    !tcp_peer_is_proven(req, dst)) {
O
Octavian Purdila 已提交
6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391
			/* Without syncookies last quarter of
			 * backlog is filled with destinations,
			 * proven to be alive.
			 * It means that we continue to communicate
			 * to destinations, already remembered
			 * to the moment of synflood.
			 */
			pr_drop_req(req, ntohs(tcp_hdr(skb)->source),
				    rsk_ops->family);
			goto drop_and_release;
		}

6392
		isn = af_ops->init_seq(skb);
O
Octavian Purdila 已提交
6393 6394
	}

6395 6396 6397 6398 6399 6400 6401 6402 6403
	tcp_ecn_create_request(req, skb, sk, dst);

	if (want_cookie) {
		isn = cookie_init_sequence(af_ops, sk, skb, &req->mss);
		req->cookie_ts = tmp_opt.tstamp_ok;
		if (!tmp_opt.tstamp_ok)
			inet_rsk(req)->ecn_ok = 0;
	}

O
Octavian Purdila 已提交
6404
	tcp_rsk(req)->snt_isn = isn;
6405
	tcp_rsk(req)->txhash = net_tx_rndhash();
O
Octavian Purdila 已提交
6406
	tcp_openreq_init_rwin(req, sk, dst);
6407 6408
	if (!want_cookie) {
		tcp_reqsk_record_syn(sk, req, skb);
6409
		fastopen_sk = tcp_try_fastopen(sk, skb, req, &foc, dst);
6410
	}
6411
	if (fastopen_sk) {
6412
		af_ops->send_synack(fastopen_sk, dst, &fl, req,
6413
				    &foc, TCP_SYNACK_FASTOPEN);
6414 6415 6416 6417
		/* Add the child socket directly into the accept queue */
		inet_csk_reqsk_queue_add(sk, req, fastopen_sk);
		sk->sk_data_ready(sk);
		bh_unlock_sock(fastopen_sk);
6418 6419
		sock_put(fastopen_sk);
	} else {
6420
		tcp_rsk(req)->tfo_listener = false;
6421
		if (!want_cookie)
6422 6423
			inet_csk_reqsk_queue_hash_add(sk, req,
				tcp_timeout_init((struct sock *)req));
6424 6425 6426
		af_ops->send_synack(sk, dst, &fl, req, &foc,
				    !want_cookie ? TCP_SYNACK_NORMAL :
						   TCP_SYNACK_COOKIE);
6427 6428 6429 6430
		if (want_cookie) {
			reqsk_free(req);
			return 0;
		}
O
Octavian Purdila 已提交
6431
	}
6432
	reqsk_put(req);
O
Octavian Purdila 已提交
6433 6434 6435 6436 6437 6438 6439
	return 0;

drop_and_release:
	dst_release(dst);
drop_and_free:
	reqsk_free(req);
drop:
6440
	tcp_listendrop(sk);
O
Octavian Purdila 已提交
6441 6442 6443
	return 0;
}
EXPORT_SYMBOL(tcp_conn_request);