tcp_ipv4.c 62.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		Implementation of the Transmission Control Protocol(TCP).
 *
 *		IPv4 specific functions
 *
 *
 *		code split from:
 *		linux/ipv4/tcp.c
 *		linux/ipv4/tcp_input.c
 *		linux/ipv4/tcp_output.c
 *
 *		See tcp.c for author information
 *
 *	This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

/*
 * Changes:
 *		David S. Miller	:	New socket lookup architecture.
 *					This code is dedicated to John Dyson.
 *		David S. Miller :	Change semantics of established hash,
 *					half is devoted to TIME_WAIT sockets
 *					and the rest go in the other half.
 *		Andi Kleen :		Add support for syncookies and fixed
 *					some bugs: ip options weren't passed to
 *					the TCP layer, missed a check for an
 *					ACK bit.
 *		Andi Kleen :		Implemented fast path mtu discovery.
 *	     				Fixed many serious bugs in the
37
 *					request_sock handling and moved
L
Linus Torvalds 已提交
38 39
 *					most of it into the af independent code.
 *					Added tail drop and some other bugfixes.
S
Stephen Hemminger 已提交
40
 *					Added new listen semantics.
L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48 49 50 51 52
 *		Mike McLagan	:	Routing by source
 *	Juan Jose Ciarlante:		ip_dynaddr bits
 *		Andi Kleen:		various fixes.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year
 *					coma.
 *	Andi Kleen		:	Fix new listen.
 *	Andi Kleen		:	Fix accept error reporting.
 *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
 *	Alexey Kuznetsov		allow both IPv4 and IPv6 sockets to bind
 *					a single port at the same time.
 */

53
#define pr_fmt(fmt) "TCP: " fmt
L
Linus Torvalds 已提交
54

H
Herbert Xu 已提交
55
#include <linux/bottom_half.h>
L
Linus Torvalds 已提交
56 57 58 59 60 61 62 63
#include <linux/types.h>
#include <linux/fcntl.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/cache.h>
#include <linux/jhash.h>
#include <linux/init.h>
#include <linux/times.h>
64
#include <linux/slab.h>
L
Linus Torvalds 已提交
65

66
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
67
#include <net/icmp.h>
68
#include <net/inet_hashtables.h>
L
Linus Torvalds 已提交
69
#include <net/tcp.h>
70
#include <net/transp_v6.h>
L
Linus Torvalds 已提交
71 72
#include <net/ipv6.h>
#include <net/inet_common.h>
73
#include <net/timewait_sock.h>
L
Linus Torvalds 已提交
74
#include <net/xfrm.h>
C
Chris Leech 已提交
75
#include <net/netdma.h>
76
#include <net/secure_seq.h>
G
Glauber Costa 已提交
77
#include <net/tcp_memcontrol.h>
78
#include <net/busy_poll.h>
L
Linus Torvalds 已提交
79 80 81 82 83 84 85

#include <linux/inet.h>
#include <linux/ipv6.h>
#include <linux/stddef.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>

86 87 88
#include <linux/crypto.h>
#include <linux/scatterlist.h>

89 90
int sysctl_tcp_tw_reuse __read_mostly;
int sysctl_tcp_low_latency __read_mostly;
E
Eric Dumazet 已提交
91
EXPORT_SYMBOL(sysctl_tcp_low_latency);
L
Linus Torvalds 已提交
92

93
#ifdef CONFIG_TCP_MD5SIG
E
Eric Dumazet 已提交
94
static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
E
Eric Dumazet 已提交
95
			       __be32 daddr, __be32 saddr, const struct tcphdr *th);
96 97
#endif

98
struct inet_hashinfo tcp_hashinfo;
E
Eric Dumazet 已提交
99
EXPORT_SYMBOL(tcp_hashinfo);
L
Linus Torvalds 已提交
100

101
static  __u32 tcp_v4_init_sequence(const struct sk_buff *skb)
L
Linus Torvalds 已提交
102
{
103 104
	return secure_tcp_sequence_number(ip_hdr(skb)->daddr,
					  ip_hdr(skb)->saddr,
105 106
					  tcp_hdr(skb)->dest,
					  tcp_hdr(skb)->source);
L
Linus Torvalds 已提交
107 108
}

109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
{
	const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
	struct tcp_sock *tp = tcp_sk(sk);

	/* With PAWS, it is safe from the viewpoint
	   of data integrity. Even without PAWS it is safe provided sequence
	   spaces do not overlap i.e. at data rates <= 80Mbit/sec.

	   Actually, the idea is close to VJ's one, only timestamp cache is
	   held not per host, but per port pair and TW bucket is used as state
	   holder.

	   If TW bucket has been already destroyed we fall back to VJ's scheme
	   and use initial timestamp retrieved from peer table.
	 */
	if (tcptw->tw_ts_recent_stamp &&
	    (twp == NULL || (sysctl_tcp_tw_reuse &&
127
			     get_seconds() - tcptw->tw_ts_recent_stamp > 1))) {
128 129 130 131 132 133 134 135 136 137 138 139 140
		tp->write_seq = tcptw->tw_snd_nxt + 65535 + 2;
		if (tp->write_seq == 0)
			tp->write_seq = 1;
		tp->rx_opt.ts_recent	   = tcptw->tw_ts_recent;
		tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
		sock_hold(sktw);
		return 1;
	}

	return 0;
}
EXPORT_SYMBOL_GPL(tcp_twsk_unique);

L
Linus Torvalds 已提交
141 142 143
/* This will initiate an outgoing connection. */
int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
{
144
	struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
L
Linus Torvalds 已提交
145 146
	struct inet_sock *inet = inet_sk(sk);
	struct tcp_sock *tp = tcp_sk(sk);
147
	__be16 orig_sport, orig_dport;
148
	__be32 daddr, nexthop;
149
	struct flowi4 *fl4;
150
	struct rtable *rt;
L
Linus Torvalds 已提交
151
	int err;
152
	struct ip_options_rcu *inet_opt;
L
Linus Torvalds 已提交
153 154 155 156 157 158 159 160

	if (addr_len < sizeof(struct sockaddr_in))
		return -EINVAL;

	if (usin->sin_family != AF_INET)
		return -EAFNOSUPPORT;

	nexthop = daddr = usin->sin_addr.s_addr;
161 162 163
	inet_opt = rcu_dereference_protected(inet->inet_opt,
					     sock_owned_by_user(sk));
	if (inet_opt && inet_opt->opt.srr) {
L
Linus Torvalds 已提交
164 165
		if (!daddr)
			return -EINVAL;
166
		nexthop = inet_opt->opt.faddr;
L
Linus Torvalds 已提交
167 168
	}

169 170
	orig_sport = inet->inet_sport;
	orig_dport = usin->sin_port;
171 172
	fl4 = &inet->cork.fl.u.ip4;
	rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
173 174
			      RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
			      IPPROTO_TCP,
175
			      orig_sport, orig_dport, sk);
176 177 178
	if (IS_ERR(rt)) {
		err = PTR_ERR(rt);
		if (err == -ENETUNREACH)
179
			IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
180
		return err;
181
	}
L
Linus Torvalds 已提交
182 183 184 185 186 187

	if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
		ip_rt_put(rt);
		return -ENETUNREACH;
	}

188
	if (!inet_opt || !inet_opt->opt.srr)
189
		daddr = fl4->daddr;
L
Linus Torvalds 已提交
190

E
Eric Dumazet 已提交
191
	if (!inet->inet_saddr)
192
		inet->inet_saddr = fl4->saddr;
E
Eric Dumazet 已提交
193
	inet->inet_rcv_saddr = inet->inet_saddr;
L
Linus Torvalds 已提交
194

E
Eric Dumazet 已提交
195
	if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
L
Linus Torvalds 已提交
196 197 198
		/* Reset inherited state */
		tp->rx_opt.ts_recent	   = 0;
		tp->rx_opt.ts_recent_stamp = 0;
P
Pavel Emelyanov 已提交
199 200
		if (likely(!tp->repair))
			tp->write_seq	   = 0;
L
Linus Torvalds 已提交
201 202
	}

203
	if (tcp_death_row.sysctl_tw_recycle &&
204 205
	    !tp->rx_opt.ts_recent_stamp && fl4->daddr == daddr)
		tcp_fetch_timewait_stamp(sk, &rt->dst);
L
Linus Torvalds 已提交
206

E
Eric Dumazet 已提交
207 208
	inet->inet_dport = usin->sin_port;
	inet->inet_daddr = daddr;
L
Linus Torvalds 已提交
209

210 211
	inet_set_txhash(sk);

212
	inet_csk(sk)->icsk_ext_hdr_len = 0;
213 214
	if (inet_opt)
		inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
L
Linus Torvalds 已提交
215

216
	tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
L
Linus Torvalds 已提交
217 218 219 220 221 222 223

	/* Socket identity is still unknown (sport may be zero).
	 * However we set state to SYN-SENT and not releasing socket
	 * lock select source port, enter ourselves into the hash tables and
	 * complete initialization after this.
	 */
	tcp_set_state(sk, TCP_SYN_SENT);
224
	err = inet_hash_connect(&tcp_death_row, sk);
L
Linus Torvalds 已提交
225 226 227
	if (err)
		goto failure;

228
	rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
229 230 231 232
			       inet->inet_sport, inet->inet_dport, sk);
	if (IS_ERR(rt)) {
		err = PTR_ERR(rt);
		rt = NULL;
L
Linus Torvalds 已提交
233
		goto failure;
234
	}
L
Linus Torvalds 已提交
235
	/* OK, now commit destination to socket.  */
236
	sk->sk_gso_type = SKB_GSO_TCPV4;
237
	sk_setup_caps(sk, &rt->dst);
L
Linus Torvalds 已提交
238

P
Pavel Emelyanov 已提交
239
	if (!tp->write_seq && likely(!tp->repair))
E
Eric Dumazet 已提交
240 241 242
		tp->write_seq = secure_tcp_sequence_number(inet->inet_saddr,
							   inet->inet_daddr,
							   inet->inet_sport,
L
Linus Torvalds 已提交
243 244
							   usin->sin_port);

E
Eric Dumazet 已提交
245
	inet->inet_id = tp->write_seq ^ jiffies;
L
Linus Torvalds 已提交
246

A
Andrey Vagin 已提交
247
	err = tcp_connect(sk);
P
Pavel Emelyanov 已提交
248

L
Linus Torvalds 已提交
249 250 251 252 253 254 255
	rt = NULL;
	if (err)
		goto failure;

	return 0;

failure:
256 257 258 259
	/*
	 * This unhashes the socket and releases the local port,
	 * if necessary.
	 */
L
Linus Torvalds 已提交
260 261 262
	tcp_set_state(sk, TCP_CLOSE);
	ip_rt_put(rt);
	sk->sk_route_caps = 0;
E
Eric Dumazet 已提交
263
	inet->inet_dport = 0;
L
Linus Torvalds 已提交
264 265
	return err;
}
E
Eric Dumazet 已提交
266
EXPORT_SYMBOL(tcp_v4_connect);
L
Linus Torvalds 已提交
267 268

/*
269 270 271
 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
 * It can be called through tcp_release_cb() if socket was owned by user
 * at the time tcp_v4_err() was called to handle ICMP message.
L
Linus Torvalds 已提交
272
 */
273
void tcp_v4_mtu_reduced(struct sock *sk)
L
Linus Torvalds 已提交
274 275 276
{
	struct dst_entry *dst;
	struct inet_sock *inet = inet_sk(sk);
277
	u32 mtu = tcp_sk(sk)->mtu_info;
L
Linus Torvalds 已提交
278

279 280
	dst = inet_csk_update_pmtu(sk, mtu);
	if (!dst)
L
Linus Torvalds 已提交
281 282 283 284 285 286 287 288 289 290 291
		return;

	/* Something is about to be wrong... Remember soft error
	 * for the case, if this connection will not able to recover.
	 */
	if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
		sk->sk_err_soft = EMSGSIZE;

	mtu = dst_mtu(dst);

	if (inet->pmtudisc != IP_PMTUDISC_DONT &&
292
	    ip_sk_accept_pmtu(sk) &&
293
	    inet_csk(sk)->icsk_pmtu_cookie > mtu) {
L
Linus Torvalds 已提交
294 295 296 297 298 299 300 301 302 303
		tcp_sync_mss(sk, mtu);

		/* Resend the TCP packet because it's
		 * clear that the old packet has been
		 * dropped. This is the new "fast" path mtu
		 * discovery.
		 */
		tcp_simple_retransmit(sk);
	} /* else let the usual retransmit timer handle it */
}
304
EXPORT_SYMBOL(tcp_v4_mtu_reduced);
L
Linus Torvalds 已提交
305

306 307 308 309
static void do_redirect(struct sk_buff *skb, struct sock *sk)
{
	struct dst_entry *dst = __sk_dst_check(sk, 0);

310
	if (dst)
311
		dst->ops->redirect(dst, sk, skb);
312 313
}

L
Linus Torvalds 已提交
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
/*
 * This routine is called by the ICMP module when it gets some
 * sort of error condition.  If err < 0 then the socket should
 * be closed and the error returned to the user.  If err > 0
 * it's just the icmp type << 8 | icmp code.  After adjustment
 * header points to the first 8 bytes of the tcp header.  We need
 * to find the appropriate port.
 *
 * The locking strategy used here is very "optimistic". When
 * someone else accesses the socket the ICMP is just dropped
 * and for some paths there is no check at all.
 * A more general error queue to queue errors for later handling
 * is probably better.
 *
 */

330
void tcp_v4_err(struct sk_buff *icmp_skb, u32 info)
L
Linus Torvalds 已提交
331
{
332
	const struct iphdr *iph = (const struct iphdr *)icmp_skb->data;
333
	struct tcphdr *th = (struct tcphdr *)(icmp_skb->data + (iph->ihl << 2));
334
	struct inet_connection_sock *icsk;
L
Linus Torvalds 已提交
335 336
	struct tcp_sock *tp;
	struct inet_sock *inet;
337 338
	const int type = icmp_hdr(icmp_skb)->type;
	const int code = icmp_hdr(icmp_skb)->code;
L
Linus Torvalds 已提交
339
	struct sock *sk;
340
	struct sk_buff *skb;
341 342
	struct request_sock *fastopen;
	__u32 seq, snd_una;
343
	__u32 remaining;
L
Linus Torvalds 已提交
344
	int err;
345
	struct net *net = dev_net(icmp_skb->dev);
L
Linus Torvalds 已提交
346

347
	sk = inet_lookup(net, &tcp_hashinfo, iph->daddr, th->dest,
348
			iph->saddr, th->source, inet_iif(icmp_skb));
L
Linus Torvalds 已提交
349
	if (!sk) {
350
		ICMP_INC_STATS_BH(net, ICMP_MIB_INERRORS);
L
Linus Torvalds 已提交
351 352 353
		return;
	}
	if (sk->sk_state == TCP_TIME_WAIT) {
354
		inet_twsk_put(inet_twsk(sk));
L
Linus Torvalds 已提交
355 356 357 358 359 360
		return;
	}

	bh_lock_sock(sk);
	/* If too many ICMPs get dropped on busy
	 * servers this needs to be solved differently.
361 362
	 * We do take care of PMTU discovery (RFC1191) special case :
	 * we can receive locally generated ICMP messages while socket is held.
L
Linus Torvalds 已提交
363
	 */
364 365 366 367
	if (sock_owned_by_user(sk)) {
		if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
			NET_INC_STATS_BH(net, LINUX_MIB_LOCKDROPPEDICMPS);
	}
L
Linus Torvalds 已提交
368 369 370
	if (sk->sk_state == TCP_CLOSE)
		goto out;

371 372 373 374 375
	if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
		NET_INC_STATS_BH(net, LINUX_MIB_TCPMINTTLDROP);
		goto out;
	}

376
	icsk = inet_csk(sk);
L
Linus Torvalds 已提交
377 378
	tp = tcp_sk(sk);
	seq = ntohl(th->seq);
379 380 381
	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
	fastopen = tp->fastopen_rsk;
	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
L
Linus Torvalds 已提交
382
	if (sk->sk_state != TCP_LISTEN &&
383
	    !between(seq, snd_una, tp->snd_nxt)) {
384
		NET_INC_STATS_BH(net, LINUX_MIB_OUTOFWINDOWICMPS);
L
Linus Torvalds 已提交
385 386 387 388
		goto out;
	}

	switch (type) {
389 390 391
	case ICMP_REDIRECT:
		do_redirect(icmp_skb, sk);
		goto out;
L
Linus Torvalds 已提交
392 393 394 395 396 397 398 399 400 401 402
	case ICMP_SOURCE_QUENCH:
		/* Just silently ignore these. */
		goto out;
	case ICMP_PARAMETERPROB:
		err = EPROTO;
		break;
	case ICMP_DEST_UNREACH:
		if (code > NR_ICMP_UNREACH)
			goto out;

		if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
403 404 405 406 407 408 409
			/* We are not interested in TCP_LISTEN and open_requests
			 * (SYN-ACKs send out by Linux are always <576bytes so
			 * they should go through unfragmented).
			 */
			if (sk->sk_state == TCP_LISTEN)
				goto out;

410
			tp->mtu_info = info;
411
			if (!sock_owned_by_user(sk)) {
412
				tcp_v4_mtu_reduced(sk);
413 414 415 416
			} else {
				if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &tp->tsq_flags))
					sock_hold(sk);
			}
L
Linus Torvalds 已提交
417 418 419 420
			goto out;
		}

		err = icmp_err_convert[code].errno;
421 422 423 424 425
		/* check if icmp_skb allows revert of backoff
		 * (see draft-zimmermann-tcp-lcd) */
		if (code != ICMP_NET_UNREACH && code != ICMP_HOST_UNREACH)
			break;
		if (seq != tp->snd_una  || !icsk->icsk_retransmits ||
426
		    !icsk->icsk_backoff || fastopen)
427 428
			break;

429 430 431
		if (sock_owned_by_user(sk))
			break;

432
		icsk->icsk_backoff--;
433
		inet_csk(sk)->icsk_rto = (tp->srtt_us ? __tcp_set_rto(tp) :
434
			TCP_TIMEOUT_INIT) << icsk->icsk_backoff;
435 436 437 438 439
		tcp_bound_rto(sk);

		skb = tcp_write_queue_head(sk);
		BUG_ON(!skb);

E
Eric Dumazet 已提交
440 441 442
		remaining = icsk->icsk_rto -
			    min(icsk->icsk_rto,
				tcp_time_stamp - tcp_skb_timestamp(skb));
443 444 445 446 447 448 449 450 451 452

		if (remaining) {
			inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
						  remaining, TCP_RTO_MAX);
		} else {
			/* RTO revert clocked out retransmission.
			 * Will retransmit now */
			tcp_retransmit_timer(sk);
		}

L
Linus Torvalds 已提交
453 454 455 456 457 458 459 460 461
		break;
	case ICMP_TIME_EXCEEDED:
		err = EHOSTUNREACH;
		break;
	default:
		goto out;
	}

	switch (sk->sk_state) {
462
		struct request_sock *req, **prev;
L
Linus Torvalds 已提交
463 464 465 466
	case TCP_LISTEN:
		if (sock_owned_by_user(sk))
			goto out;

467 468
		req = inet_csk_search_req(sk, &prev, th->dest,
					  iph->daddr, iph->saddr);
L
Linus Torvalds 已提交
469 470 471 472 473 474
		if (!req)
			goto out;

		/* ICMPs are not backlogged, hence we cannot get
		   an established socket here.
		 */
475
		WARN_ON(req->sk);
L
Linus Torvalds 已提交
476

477
		if (seq != tcp_rsk(req)->snt_isn) {
478
			NET_INC_STATS_BH(net, LINUX_MIB_OUTOFWINDOWICMPS);
L
Linus Torvalds 已提交
479 480 481 482 483 484 485 486 487
			goto out;
		}

		/*
		 * Still in SYN_RECV, just remove it silently.
		 * There is no good way to pass the error to the newly
		 * created socket, and POSIX does not want network
		 * errors returned from accept().
		 */
488
		inet_csk_reqsk_queue_drop(sk, req, prev);
489
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS);
L
Linus Torvalds 已提交
490 491 492
		goto out;

	case TCP_SYN_SENT:
493 494 495 496 497 498 499
	case TCP_SYN_RECV:
		/* Only in fast or simultaneous open. If a fast open socket is
		 * is already accepted it is treated as a connected one below.
		 */
		if (fastopen && fastopen->sk == NULL)
			break;

L
Linus Torvalds 已提交
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
		if (!sock_owned_by_user(sk)) {
			sk->sk_err = err;

			sk->sk_error_report(sk);

			tcp_done(sk);
		} else {
			sk->sk_err_soft = err;
		}
		goto out;
	}

	/* If we've already connected we will keep trying
	 * until we time out, or the user gives up.
	 *
	 * rfc1122 4.2.3.9 allows to consider as hard errors
	 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
	 * but it is obsoleted by pmtu discovery).
	 *
	 * Note, that in modern internet, where routing is unreliable
	 * and in each dark corner broken firewalls sit, sending random
	 * errors ordered by their masters even this two messages finally lose
	 * their original sense (even Linux sends invalid PORT_UNREACHs)
	 *
	 * Now we are in compliance with RFCs.
	 *							--ANK (980905)
	 */

	inet = inet_sk(sk);
	if (!sock_owned_by_user(sk) && inet->recverr) {
		sk->sk_err = err;
		sk->sk_error_report(sk);
	} else	{ /* Only an error on timeout */
		sk->sk_err_soft = err;
	}

out:
	bh_unlock_sock(sk);
	sock_put(sk);
}

541
void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
L
Linus Torvalds 已提交
542
{
543
	struct tcphdr *th = tcp_hdr(skb);
L
Linus Torvalds 已提交
544

545
	if (skb->ip_summed == CHECKSUM_PARTIAL) {
546
		th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
547
		skb->csum_start = skb_transport_header(skb) - skb->head;
A
Al Viro 已提交
548
		skb->csum_offset = offsetof(struct tcphdr, check);
L
Linus Torvalds 已提交
549
	} else {
550
		th->check = tcp_v4_check(skb->len, saddr, daddr,
551
					 csum_partial(th,
L
Linus Torvalds 已提交
552 553 554 555 556
						      th->doff << 2,
						      skb->csum));
	}
}

557
/* This routine computes an IPv4 TCP checksum. */
558
void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
559
{
560
	const struct inet_sock *inet = inet_sk(sk);
561 562 563

	__tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
}
E
Eric Dumazet 已提交
564
EXPORT_SYMBOL(tcp_v4_send_check);
565

L
Linus Torvalds 已提交
566 567 568 569 570 571 572 573 574 575 576 577 578
/*
 *	This routine will send an RST to the other tcp.
 *
 *	Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
 *		      for reset.
 *	Answer: if a packet caused RST, it is not for a socket
 *		existing in our system, if it is matched to a socket,
 *		it is just duplicate segment or bug in other side's TCP.
 *		So that we build reply only basing on parameters
 *		arrived with segment.
 *	Exception: precedence violation. We do not implement it in any case.
 */

579
static void tcp_v4_send_reset(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
580
{
581
	const struct tcphdr *th = tcp_hdr(skb);
582 583 584
	struct {
		struct tcphdr th;
#ifdef CONFIG_TCP_MD5SIG
585
		__be32 opt[(TCPOLEN_MD5SIG_ALIGNED >> 2)];
586 587
#endif
	} rep;
L
Linus Torvalds 已提交
588
	struct ip_reply_arg arg;
589 590
#ifdef CONFIG_TCP_MD5SIG
	struct tcp_md5sig_key *key;
591 592 593 594
	const __u8 *hash_location = NULL;
	unsigned char newhash[16];
	int genhash;
	struct sock *sk1 = NULL;
595
#endif
596
	struct net *net;
L
Linus Torvalds 已提交
597 598 599 600 601

	/* Never send a reset in response to a reset. */
	if (th->rst)
		return;

E
Eric Dumazet 已提交
602
	if (skb_rtable(skb)->rt_type != RTN_LOCAL)
L
Linus Torvalds 已提交
603 604 605
		return;

	/* Swap the send and the receive. */
606 607 608 609 610
	memset(&rep, 0, sizeof(rep));
	rep.th.dest   = th->source;
	rep.th.source = th->dest;
	rep.th.doff   = sizeof(struct tcphdr) / 4;
	rep.th.rst    = 1;
L
Linus Torvalds 已提交
611 612

	if (th->ack) {
613
		rep.th.seq = th->ack_seq;
L
Linus Torvalds 已提交
614
	} else {
615 616 617
		rep.th.ack = 1;
		rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
				       skb->len - (th->doff << 2));
L
Linus Torvalds 已提交
618 619
	}

620
	memset(&arg, 0, sizeof(arg));
621 622 623 624
	arg.iov[0].iov_base = (unsigned char *)&rep;
	arg.iov[0].iov_len  = sizeof(rep.th);

#ifdef CONFIG_TCP_MD5SIG
625 626 627 628 629 630 631 632 633 634
	hash_location = tcp_parse_md5sig_option(th);
	if (!sk && hash_location) {
		/*
		 * active side is lost. Try to find listening socket through
		 * source port, and then find md5 key through listening socket.
		 * we are not loose security here:
		 * Incoming packet is checked with md5 hash with finding key,
		 * no RST generated if md5 hash doesn't match.
		 */
		sk1 = __inet_lookup_listener(dev_net(skb_dst(skb)->dev),
635 636
					     &tcp_hashinfo, ip_hdr(skb)->saddr,
					     th->source, ip_hdr(skb)->daddr,
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
					     ntohs(th->source), inet_iif(skb));
		/* don't send rst if it can't find key */
		if (!sk1)
			return;
		rcu_read_lock();
		key = tcp_md5_do_lookup(sk1, (union tcp_md5_addr *)
					&ip_hdr(skb)->saddr, AF_INET);
		if (!key)
			goto release_sk1;

		genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, NULL, skb);
		if (genhash || memcmp(hash_location, newhash, 16) != 0)
			goto release_sk1;
	} else {
		key = sk ? tcp_md5_do_lookup(sk, (union tcp_md5_addr *)
					     &ip_hdr(skb)->saddr,
					     AF_INET) : NULL;
	}

656 657 658 659 660 661 662 663 664
	if (key) {
		rep.opt[0] = htonl((TCPOPT_NOP << 24) |
				   (TCPOPT_NOP << 16) |
				   (TCPOPT_MD5SIG << 8) |
				   TCPOLEN_MD5SIG);
		/* Update length and the length the header thinks exists */
		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
		rep.th.doff = arg.iov[0].iov_len / 4;

665
		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
666 667
				     key, ip_hdr(skb)->saddr,
				     ip_hdr(skb)->daddr, &rep.th);
668 669
	}
#endif
670 671
	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
				      ip_hdr(skb)->saddr, /* XXX */
672
				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
L
Linus Torvalds 已提交
673
	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
674
	arg.flags = (sk && inet_sk(sk)->transparent) ? IP_REPLY_ARG_NOSRCCHECK : 0;
675
	/* When socket is gone, all binding information is lost.
A
Alexey Kuznetsov 已提交
676 677
	 * routing might fail in this case. No choice here, if we choose to force
	 * input interface, we will misroute in case of asymmetric route.
678
	 */
A
Alexey Kuznetsov 已提交
679 680
	if (sk)
		arg.bound_dev_if = sk->sk_bound_dev_if;
L
Linus Torvalds 已提交
681

E
Eric Dumazet 已提交
682
	net = dev_net(skb_dst(skb)->dev);
683
	arg.tos = ip_hdr(skb)->tos;
684
	ip_send_unicast_reply(net, skb, ip_hdr(skb)->saddr,
685
			      ip_hdr(skb)->daddr, &arg, arg.iov[0].iov_len);
L
Linus Torvalds 已提交
686

687 688
	TCP_INC_STATS_BH(net, TCP_MIB_OUTSEGS);
	TCP_INC_STATS_BH(net, TCP_MIB_OUTRSTS);
689 690 691 692 693 694 695 696

#ifdef CONFIG_TCP_MD5SIG
release_sk1:
	if (sk1) {
		rcu_read_unlock();
		sock_put(sk1);
	}
#endif
L
Linus Torvalds 已提交
697 698 699 700 701 702
}

/* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
   outside socket context is ugly, certainly. What can I do?
 */

703
static void tcp_v4_send_ack(struct sk_buff *skb, u32 seq, u32 ack,
704
			    u32 win, u32 tsval, u32 tsecr, int oif,
705
			    struct tcp_md5sig_key *key,
706
			    int reply_flags, u8 tos)
L
Linus Torvalds 已提交
707
{
708
	const struct tcphdr *th = tcp_hdr(skb);
L
Linus Torvalds 已提交
709 710
	struct {
		struct tcphdr th;
711
		__be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
712
#ifdef CONFIG_TCP_MD5SIG
713
			   + (TCPOLEN_MD5SIG_ALIGNED >> 2)
714 715
#endif
			];
L
Linus Torvalds 已提交
716 717
	} rep;
	struct ip_reply_arg arg;
E
Eric Dumazet 已提交
718
	struct net *net = dev_net(skb_dst(skb)->dev);
L
Linus Torvalds 已提交
719 720

	memset(&rep.th, 0, sizeof(struct tcphdr));
721
	memset(&arg, 0, sizeof(arg));
L
Linus Torvalds 已提交
722 723 724

	arg.iov[0].iov_base = (unsigned char *)&rep;
	arg.iov[0].iov_len  = sizeof(rep.th);
725
	if (tsecr) {
726 727 728
		rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
				   (TCPOPT_TIMESTAMP << 8) |
				   TCPOLEN_TIMESTAMP);
729 730
		rep.opt[1] = htonl(tsval);
		rep.opt[2] = htonl(tsecr);
731
		arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
L
Linus Torvalds 已提交
732 733 734 735 736 737 738 739 740 741 742
	}

	/* Swap the send and the receive. */
	rep.th.dest    = th->source;
	rep.th.source  = th->dest;
	rep.th.doff    = arg.iov[0].iov_len / 4;
	rep.th.seq     = htonl(seq);
	rep.th.ack_seq = htonl(ack);
	rep.th.ack     = 1;
	rep.th.window  = htons(win);

743 744
#ifdef CONFIG_TCP_MD5SIG
	if (key) {
745
		int offset = (tsecr) ? 3 : 0;
746 747 748 749 750 751 752 753

		rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
					  (TCPOPT_NOP << 16) |
					  (TCPOPT_MD5SIG << 8) |
					  TCPOLEN_MD5SIG);
		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
		rep.th.doff = arg.iov[0].iov_len/4;

754
		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
755 756
				    key, ip_hdr(skb)->saddr,
				    ip_hdr(skb)->daddr, &rep.th);
757 758
	}
#endif
759
	arg.flags = reply_flags;
760 761
	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
				      ip_hdr(skb)->saddr, /* XXX */
L
Linus Torvalds 已提交
762 763
				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
764 765
	if (oif)
		arg.bound_dev_if = oif;
766
	arg.tos = tos;
767
	ip_send_unicast_reply(net, skb, ip_hdr(skb)->saddr,
768
			      ip_hdr(skb)->daddr, &arg, arg.iov[0].iov_len);
L
Linus Torvalds 已提交
769

770
	TCP_INC_STATS_BH(net, TCP_MIB_OUTSEGS);
L
Linus Torvalds 已提交
771 772 773 774
}

static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
{
775
	struct inet_timewait_sock *tw = inet_twsk(sk);
776
	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
L
Linus Torvalds 已提交
777

778
	tcp_v4_send_ack(skb, tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
779
			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
780
			tcp_time_stamp + tcptw->tw_ts_offset,
781 782
			tcptw->tw_ts_recent,
			tw->tw_bound_dev_if,
783
			tcp_twsk_md5_key(tcptw),
784 785
			tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
			tw->tw_tos
786
			);
L
Linus Torvalds 已提交
787

788
	inet_twsk_put(tw);
L
Linus Torvalds 已提交
789 790
}

791
static void tcp_v4_reqsk_send_ack(struct sock *sk, struct sk_buff *skb,
792
				  struct request_sock *req)
L
Linus Torvalds 已提交
793
{
794 795 796 797 798 799
	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
	 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
	 */
	tcp_v4_send_ack(skb, (sk->sk_state == TCP_LISTEN) ?
			tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
			tcp_rsk(req)->rcv_nxt, req->rcv_wnd,
800
			tcp_time_stamp,
801 802
			req->ts_recent,
			0,
E
Eric Dumazet 已提交
803 804
			tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&ip_hdr(skb)->daddr,
					  AF_INET),
805 806
			inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
			ip_hdr(skb)->tos);
L
Linus Torvalds 已提交
807 808 809
}

/*
810
 *	Send a SYN-ACK after having received a SYN.
811
 *	This still operates on a request_sock only, not on a big
L
Linus Torvalds 已提交
812 813
 *	socket.
 */
814
static int tcp_v4_send_synack(struct sock *sk, struct dst_entry *dst,
815
			      struct flowi *fl,
816
			      struct request_sock *req,
817 818
			      u16 queue_mapping,
			      struct tcp_fastopen_cookie *foc)
L
Linus Torvalds 已提交
819
{
820
	const struct inet_request_sock *ireq = inet_rsk(req);
821
	struct flowi4 fl4;
L
Linus Torvalds 已提交
822
	int err = -1;
823
	struct sk_buff *skb;
L
Linus Torvalds 已提交
824 825

	/* First, grab a route. */
826
	if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
827
		return -1;
L
Linus Torvalds 已提交
828

829
	skb = tcp_make_synack(sk, dst, req, foc);
L
Linus Torvalds 已提交
830 831

	if (skb) {
832
		__tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
L
Linus Torvalds 已提交
833

834
		skb_set_queue_mapping(skb, queue_mapping);
835 836
		err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
					    ireq->ir_rmt_addr,
837
					    ireq->opt);
838
		err = net_xmit_eval(err);
L
Linus Torvalds 已提交
839 840 841 842 843 844
	}

	return err;
}

/*
845
 *	IPv4 request_sock destructor.
L
Linus Torvalds 已提交
846
 */
847
static void tcp_v4_reqsk_destructor(struct request_sock *req)
L
Linus Torvalds 已提交
848
{
J
Jesper Juhl 已提交
849
	kfree(inet_rsk(req)->opt);
L
Linus Torvalds 已提交
850 851
}

852
/*
E
Eric Dumazet 已提交
853
 * Return true if a syncookie should be sent
854
 */
E
Eric Dumazet 已提交
855
bool tcp_syn_flood_action(struct sock *sk,
856 857
			 const struct sk_buff *skb,
			 const char *proto)
L
Linus Torvalds 已提交
858
{
859
	const char *msg = "Dropping request";
E
Eric Dumazet 已提交
860
	bool want_cookie = false;
861 862
	struct listen_sock *lopt;

863
#ifdef CONFIG_SYN_COOKIES
864
	if (sysctl_tcp_syncookies) {
865
		msg = "Sending cookies";
E
Eric Dumazet 已提交
866
		want_cookie = true;
867 868
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES);
	} else
869
#endif
870 871 872
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP);

	lopt = inet_csk(sk)->icsk_accept_queue.listen_opt;
873
	if (!lopt->synflood_warned && sysctl_tcp_syncookies != 2) {
874
		lopt->synflood_warned = 1;
875
		pr_info("%s: Possible SYN flooding on port %d. %s.  Check SNMP counters.\n",
876 877 878
			proto, ntohs(tcp_hdr(skb)->dest), msg);
	}
	return want_cookie;
879
}
880
EXPORT_SYMBOL(tcp_syn_flood_action);
L
Linus Torvalds 已提交
881 882

/*
883
 * Save and compile IPv4 options into the request_sock if needed.
L
Linus Torvalds 已提交
884
 */
885
static struct ip_options_rcu *tcp_v4_save_options(struct sk_buff *skb)
L
Linus Torvalds 已提交
886
{
887 888
	const struct ip_options *opt = &(IPCB(skb)->opt);
	struct ip_options_rcu *dopt = NULL;
L
Linus Torvalds 已提交
889 890

	if (opt && opt->optlen) {
891 892
		int opt_size = sizeof(*dopt) + opt->optlen;

L
Linus Torvalds 已提交
893 894
		dopt = kmalloc(opt_size, GFP_ATOMIC);
		if (dopt) {
895
			if (ip_options_echo(&dopt->opt, skb)) {
L
Linus Torvalds 已提交
896 897 898 899 900 901 902 903
				kfree(dopt);
				dopt = NULL;
			}
		}
	}
	return dopt;
}

904 905 906 907 908 909 910 911
#ifdef CONFIG_TCP_MD5SIG
/*
 * RFC2385 MD5 checksumming requires a mapping of
 * IP address->MD5 Key.
 * We need to maintain these in the sk structure.
 */

/* Find the Key structure for an address.  */
E
Eric Dumazet 已提交
912 913 914
struct tcp_md5sig_key *tcp_md5_do_lookup(struct sock *sk,
					 const union tcp_md5_addr *addr,
					 int family)
915 916
{
	struct tcp_sock *tp = tcp_sk(sk);
E
Eric Dumazet 已提交
917 918
	struct tcp_md5sig_key *key;
	unsigned int size = sizeof(struct in_addr);
919
	struct tcp_md5sig_info *md5sig;
920

921 922
	/* caller either holds rcu_read_lock() or socket lock */
	md5sig = rcu_dereference_check(tp->md5sig_info,
923 924
				       sock_owned_by_user(sk) ||
				       lockdep_is_held(&sk->sk_lock.slock));
925
	if (!md5sig)
926
		return NULL;
E
Eric Dumazet 已提交
927 928 929 930
#if IS_ENABLED(CONFIG_IPV6)
	if (family == AF_INET6)
		size = sizeof(struct in6_addr);
#endif
931
	hlist_for_each_entry_rcu(key, &md5sig->head, node) {
E
Eric Dumazet 已提交
932 933 934 935
		if (key->family != family)
			continue;
		if (!memcmp(&key->addr, addr, size))
			return key;
936 937 938
	}
	return NULL;
}
E
Eric Dumazet 已提交
939
EXPORT_SYMBOL(tcp_md5_do_lookup);
940 941 942 943

struct tcp_md5sig_key *tcp_v4_md5_lookup(struct sock *sk,
					 struct sock *addr_sk)
{
E
Eric Dumazet 已提交
944 945 946 947
	union tcp_md5_addr *addr;

	addr = (union tcp_md5_addr *)&inet_sk(addr_sk)->inet_daddr;
	return tcp_md5_do_lookup(sk, addr, AF_INET);
948 949 950
}
EXPORT_SYMBOL(tcp_v4_md5_lookup);

A
Adrian Bunk 已提交
951 952
static struct tcp_md5sig_key *tcp_v4_reqsk_md5_lookup(struct sock *sk,
						      struct request_sock *req)
953
{
E
Eric Dumazet 已提交
954 955
	union tcp_md5_addr *addr;

956
	addr = (union tcp_md5_addr *)&inet_rsk(req)->ir_rmt_addr;
E
Eric Dumazet 已提交
957
	return tcp_md5_do_lookup(sk, addr, AF_INET);
958 959 960
}

/* This can be called on a newly created socket, from other files */
E
Eric Dumazet 已提交
961 962
int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
		   int family, const u8 *newkey, u8 newkeylen, gfp_t gfp)
963 964
{
	/* Add Key to the list */
965
	struct tcp_md5sig_key *key;
966
	struct tcp_sock *tp = tcp_sk(sk);
E
Eric Dumazet 已提交
967
	struct tcp_md5sig_info *md5sig;
968

969
	key = tcp_md5_do_lookup(sk, addr, family);
970 971
	if (key) {
		/* Pre-existing entry - just update that one. */
E
Eric Dumazet 已提交
972
		memcpy(key->key, newkey, newkeylen);
973
		key->keylen = newkeylen;
E
Eric Dumazet 已提交
974 975
		return 0;
	}
976

977 978
	md5sig = rcu_dereference_protected(tp->md5sig_info,
					   sock_owned_by_user(sk));
E
Eric Dumazet 已提交
979 980 981
	if (!md5sig) {
		md5sig = kmalloc(sizeof(*md5sig), gfp);
		if (!md5sig)
982 983
			return -ENOMEM;

E
Eric Dumazet 已提交
984 985
		sk_nocaps_add(sk, NETIF_F_GSO_MASK);
		INIT_HLIST_HEAD(&md5sig->head);
986
		rcu_assign_pointer(tp->md5sig_info, md5sig);
E
Eric Dumazet 已提交
987
	}
988

989
	key = sock_kmalloc(sk, sizeof(*key), gfp);
E
Eric Dumazet 已提交
990 991
	if (!key)
		return -ENOMEM;
992
	if (!tcp_alloc_md5sig_pool()) {
993
		sock_kfree_s(sk, key, sizeof(*key));
E
Eric Dumazet 已提交
994
		return -ENOMEM;
995
	}
E
Eric Dumazet 已提交
996 997 998 999 1000 1001 1002 1003

	memcpy(key->key, newkey, newkeylen);
	key->keylen = newkeylen;
	key->family = family;
	memcpy(&key->addr, addr,
	       (family == AF_INET6) ? sizeof(struct in6_addr) :
				      sizeof(struct in_addr));
	hlist_add_head_rcu(&key->node, &md5sig->head);
1004 1005
	return 0;
}
E
Eric Dumazet 已提交
1006
EXPORT_SYMBOL(tcp_md5_do_add);
1007

E
Eric Dumazet 已提交
1008
int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family)
1009
{
E
Eric Dumazet 已提交
1010 1011
	struct tcp_md5sig_key *key;

1012
	key = tcp_md5_do_lookup(sk, addr, family);
E
Eric Dumazet 已提交
1013 1014 1015
	if (!key)
		return -ENOENT;
	hlist_del_rcu(&key->node);
1016
	atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
E
Eric Dumazet 已提交
1017 1018
	kfree_rcu(key, rcu);
	return 0;
1019
}
E
Eric Dumazet 已提交
1020
EXPORT_SYMBOL(tcp_md5_do_del);
1021

1022
static void tcp_clear_md5_list(struct sock *sk)
1023 1024
{
	struct tcp_sock *tp = tcp_sk(sk);
E
Eric Dumazet 已提交
1025
	struct tcp_md5sig_key *key;
1026
	struct hlist_node *n;
1027
	struct tcp_md5sig_info *md5sig;
1028

1029 1030
	md5sig = rcu_dereference_protected(tp->md5sig_info, 1);

1031
	hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
E
Eric Dumazet 已提交
1032
		hlist_del_rcu(&key->node);
1033
		atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
E
Eric Dumazet 已提交
1034
		kfree_rcu(key, rcu);
1035 1036 1037
	}
}

1038 1039
static int tcp_v4_parse_md5_keys(struct sock *sk, char __user *optval,
				 int optlen)
1040 1041 1042 1043 1044 1045 1046
{
	struct tcp_md5sig cmd;
	struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;

	if (optlen < sizeof(cmd))
		return -EINVAL;

1047
	if (copy_from_user(&cmd, optval, sizeof(cmd)))
1048 1049 1050 1051 1052
		return -EFAULT;

	if (sin->sin_family != AF_INET)
		return -EINVAL;

1053
	if (!cmd.tcpm_keylen)
E
Eric Dumazet 已提交
1054 1055
		return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
				      AF_INET);
1056 1057 1058 1059

	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
		return -EINVAL;

E
Eric Dumazet 已提交
1060 1061 1062
	return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin->sin_addr.s_addr,
			      AF_INET, cmd.tcpm_key, cmd.tcpm_keylen,
			      GFP_KERNEL);
1063 1064
}

1065 1066
static int tcp_v4_md5_hash_pseudoheader(struct tcp_md5sig_pool *hp,
					__be32 daddr, __be32 saddr, int nbytes)
1067 1068
{
	struct tcp4_pseudohdr *bp;
1069
	struct scatterlist sg;
1070 1071 1072 1073

	bp = &hp->md5_blk.ip4;

	/*
1074
	 * 1. the TCP pseudo-header (in the order: source IP address,
1075 1076 1077 1078 1079 1080
	 * destination IP address, zero-padded protocol number, and
	 * segment length)
	 */
	bp->saddr = saddr;
	bp->daddr = daddr;
	bp->pad = 0;
1081
	bp->protocol = IPPROTO_TCP;
1082
	bp->len = cpu_to_be16(nbytes);
1083

1084 1085 1086 1087
	sg_init_one(&sg, bp, sizeof(*bp));
	return crypto_hash_update(&hp->md5_desc, &sg, sizeof(*bp));
}

E
Eric Dumazet 已提交
1088
static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
E
Eric Dumazet 已提交
1089
			       __be32 daddr, __be32 saddr, const struct tcphdr *th)
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
{
	struct tcp_md5sig_pool *hp;
	struct hash_desc *desc;

	hp = tcp_get_md5sig_pool();
	if (!hp)
		goto clear_hash_noput;
	desc = &hp->md5_desc;

	if (crypto_hash_init(desc))
		goto clear_hash;
	if (tcp_v4_md5_hash_pseudoheader(hp, daddr, saddr, th->doff << 2))
		goto clear_hash;
	if (tcp_md5_hash_header(hp, th))
		goto clear_hash;
	if (tcp_md5_hash_key(hp, key))
		goto clear_hash;
	if (crypto_hash_final(desc, md5_hash))
1108 1109 1110 1111
		goto clear_hash;

	tcp_put_md5sig_pool();
	return 0;
1112

1113 1114 1115 1116
clear_hash:
	tcp_put_md5sig_pool();
clear_hash_noput:
	memset(md5_hash, 0, 16);
1117
	return 1;
1118 1119
}

1120
int tcp_v4_md5_hash_skb(char *md5_hash, struct tcp_md5sig_key *key,
E
Eric Dumazet 已提交
1121 1122
			const struct sock *sk, const struct request_sock *req,
			const struct sk_buff *skb)
1123
{
1124 1125
	struct tcp_md5sig_pool *hp;
	struct hash_desc *desc;
E
Eric Dumazet 已提交
1126
	const struct tcphdr *th = tcp_hdr(skb);
1127 1128 1129
	__be32 saddr, daddr;

	if (sk) {
E
Eric Dumazet 已提交
1130 1131
		saddr = inet_sk(sk)->inet_saddr;
		daddr = inet_sk(sk)->inet_daddr;
1132
	} else if (req) {
1133 1134
		saddr = inet_rsk(req)->ir_loc_addr;
		daddr = inet_rsk(req)->ir_rmt_addr;
1135
	} else {
1136 1137 1138
		const struct iphdr *iph = ip_hdr(skb);
		saddr = iph->saddr;
		daddr = iph->daddr;
1139
	}
1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167

	hp = tcp_get_md5sig_pool();
	if (!hp)
		goto clear_hash_noput;
	desc = &hp->md5_desc;

	if (crypto_hash_init(desc))
		goto clear_hash;

	if (tcp_v4_md5_hash_pseudoheader(hp, daddr, saddr, skb->len))
		goto clear_hash;
	if (tcp_md5_hash_header(hp, th))
		goto clear_hash;
	if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
		goto clear_hash;
	if (tcp_md5_hash_key(hp, key))
		goto clear_hash;
	if (crypto_hash_final(desc, md5_hash))
		goto clear_hash;

	tcp_put_md5sig_pool();
	return 0;

clear_hash:
	tcp_put_md5sig_pool();
clear_hash_noput:
	memset(md5_hash, 0, 16);
	return 1;
1168
}
1169
EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1170

1171 1172
static bool __tcp_v4_inbound_md5_hash(struct sock *sk,
				      const struct sk_buff *skb)
1173 1174 1175 1176 1177 1178 1179 1180 1181
{
	/*
	 * This gets called for each TCP segment that arrives
	 * so we want to be efficient.
	 * We have 3 drop cases:
	 * o No MD5 hash and one expected.
	 * o MD5 hash and we're not expecting one.
	 * o MD5 hash and its wrong.
	 */
1182
	const __u8 *hash_location = NULL;
1183
	struct tcp_md5sig_key *hash_expected;
1184
	const struct iphdr *iph = ip_hdr(skb);
1185
	const struct tcphdr *th = tcp_hdr(skb);
1186 1187 1188
	int genhash;
	unsigned char newhash[16];

E
Eric Dumazet 已提交
1189 1190
	hash_expected = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&iph->saddr,
					  AF_INET);
1191
	hash_location = tcp_parse_md5sig_option(th);
1192 1193 1194

	/* We've parsed the options - do we have a hash? */
	if (!hash_expected && !hash_location)
E
Eric Dumazet 已提交
1195
		return false;
1196 1197

	if (hash_expected && !hash_location) {
1198
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
E
Eric Dumazet 已提交
1199
		return true;
1200 1201 1202
	}

	if (!hash_expected && hash_location) {
1203
		NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
E
Eric Dumazet 已提交
1204
		return true;
1205 1206 1207 1208 1209
	}

	/* Okay, so this is hash_expected and hash_location -
	 * so we need to calculate the checksum.
	 */
1210 1211 1212
	genhash = tcp_v4_md5_hash_skb(newhash,
				      hash_expected,
				      NULL, NULL, skb);
1213 1214

	if (genhash || memcmp(hash_location, newhash, 16) != 0) {
1215 1216 1217 1218 1219
		net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s\n",
				     &iph->saddr, ntohs(th->source),
				     &iph->daddr, ntohs(th->dest),
				     genhash ? " tcp_v4_calc_md5_hash failed"
				     : "");
E
Eric Dumazet 已提交
1220
		return true;
1221
	}
E
Eric Dumazet 已提交
1222
	return false;
1223 1224
}

1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
static bool tcp_v4_inbound_md5_hash(struct sock *sk, const struct sk_buff *skb)
{
	bool ret;

	rcu_read_lock();
	ret = __tcp_v4_inbound_md5_hash(sk, skb);
	rcu_read_unlock();

	return ret;
}

1236 1237
#endif

1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
static void tcp_v4_init_req(struct request_sock *req, struct sock *sk,
			    struct sk_buff *skb)
{
	struct inet_request_sock *ireq = inet_rsk(req);

	ireq->ir_loc_addr = ip_hdr(skb)->daddr;
	ireq->ir_rmt_addr = ip_hdr(skb)->saddr;
	ireq->no_srccheck = inet_sk(sk)->transparent;
	ireq->opt = tcp_v4_save_options(skb);
}

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264
static struct dst_entry *tcp_v4_route_req(struct sock *sk, struct flowi *fl,
					  const struct request_sock *req,
					  bool *strict)
{
	struct dst_entry *dst = inet_csk_route_req(sk, &fl->u.ip4, req);

	if (strict) {
		if (fl->u.ip4.daddr == inet_rsk(req)->ir_rmt_addr)
			*strict = true;
		else
			*strict = false;
	}

	return dst;
}

1265
struct request_sock_ops tcp_request_sock_ops __read_mostly = {
L
Linus Torvalds 已提交
1266
	.family		=	PF_INET,
1267
	.obj_size	=	sizeof(struct tcp_request_sock),
1268
	.rtx_syn_ack	=	tcp_rtx_synack,
1269 1270
	.send_ack	=	tcp_v4_reqsk_send_ack,
	.destructor	=	tcp_v4_reqsk_destructor,
L
Linus Torvalds 已提交
1271
	.send_reset	=	tcp_v4_send_reset,
S
stephen hemminger 已提交
1272
	.syn_ack_timeout =	tcp_syn_ack_timeout,
L
Linus Torvalds 已提交
1273 1274
};

1275
static const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1276
	.mss_clamp	=	TCP_MSS_DEFAULT,
1277
#ifdef CONFIG_TCP_MD5SIG
1278
	.md5_lookup	=	tcp_v4_reqsk_md5_lookup,
1279
	.calc_md5_hash	=	tcp_v4_md5_hash_skb,
1280
#endif
1281
	.init_req	=	tcp_v4_init_req,
1282 1283 1284
#ifdef CONFIG_SYN_COOKIES
	.cookie_init_seq =	cookie_v4_init_sequence,
#endif
1285
	.route_req	=	tcp_v4_route_req,
1286
	.init_seq	=	tcp_v4_init_sequence,
1287
	.send_synack	=	tcp_v4_send_synack,
1288
	.queue_hash_add =	inet_csk_reqsk_queue_hash_add,
1289
};
1290

L
Linus Torvalds 已提交
1291 1292 1293
int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
{
	/* Never answer to SYNs send to broadcast or multicast */
E
Eric Dumazet 已提交
1294
	if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
L
Linus Torvalds 已提交
1295 1296
		goto drop;

O
Octavian Purdila 已提交
1297 1298
	return tcp_conn_request(&tcp_request_sock_ops,
				&tcp_request_sock_ipv4_ops, sk, skb);
L
Linus Torvalds 已提交
1299 1300

drop:
1301
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS);
L
Linus Torvalds 已提交
1302 1303
	return 0;
}
E
Eric Dumazet 已提交
1304
EXPORT_SYMBOL(tcp_v4_conn_request);
L
Linus Torvalds 已提交
1305 1306 1307 1308 1309 1310 1311


/*
 * The three way handshake has completed - we got a valid synack -
 * now create the new socket.
 */
struct sock *tcp_v4_syn_recv_sock(struct sock *sk, struct sk_buff *skb,
1312
				  struct request_sock *req,
L
Linus Torvalds 已提交
1313 1314
				  struct dst_entry *dst)
{
1315
	struct inet_request_sock *ireq;
L
Linus Torvalds 已提交
1316 1317 1318
	struct inet_sock *newinet;
	struct tcp_sock *newtp;
	struct sock *newsk;
1319 1320 1321
#ifdef CONFIG_TCP_MD5SIG
	struct tcp_md5sig_key *key;
#endif
1322
	struct ip_options_rcu *inet_opt;
L
Linus Torvalds 已提交
1323 1324 1325 1326 1327 1328

	if (sk_acceptq_is_full(sk))
		goto exit_overflow;

	newsk = tcp_create_openreq_child(sk, req, skb);
	if (!newsk)
1329
		goto exit_nonewsk;
L
Linus Torvalds 已提交
1330

1331
	newsk->sk_gso_type = SKB_GSO_TCPV4;
1332
	inet_sk_rx_dst_set(newsk, skb);
L
Linus Torvalds 已提交
1333 1334 1335

	newtp		      = tcp_sk(newsk);
	newinet		      = inet_sk(newsk);
1336
	ireq		      = inet_rsk(req);
1337 1338 1339
	newinet->inet_daddr   = ireq->ir_rmt_addr;
	newinet->inet_rcv_saddr = ireq->ir_loc_addr;
	newinet->inet_saddr	      = ireq->ir_loc_addr;
1340 1341
	inet_opt	      = ireq->opt;
	rcu_assign_pointer(newinet->inet_opt, inet_opt);
1342
	ireq->opt	      = NULL;
1343
	newinet->mc_index     = inet_iif(skb);
1344
	newinet->mc_ttl	      = ip_hdr(skb)->ttl;
1345
	newinet->rcv_tos      = ip_hdr(skb)->tos;
1346
	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1347
	inet_set_txhash(newsk);
1348 1349
	if (inet_opt)
		inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
E
Eric Dumazet 已提交
1350
	newinet->inet_id = newtp->write_seq ^ jiffies;
L
Linus Torvalds 已提交
1351

E
Eric Dumazet 已提交
1352 1353 1354 1355 1356 1357 1358
	if (!dst) {
		dst = inet_csk_route_child_sock(sk, newsk, req);
		if (!dst)
			goto put_and_exit;
	} else {
		/* syncookie case : see end of cookie_v4_check() */
	}
1359 1360
	sk_setup_caps(newsk, dst);

L
Linus Torvalds 已提交
1361
	tcp_sync_mss(newsk, dst_mtu(dst));
1362
	newtp->advmss = dst_metric_advmss(dst);
1363 1364 1365 1366
	if (tcp_sk(sk)->rx_opt.user_mss &&
	    tcp_sk(sk)->rx_opt.user_mss < newtp->advmss)
		newtp->advmss = tcp_sk(sk)->rx_opt.user_mss;

L
Linus Torvalds 已提交
1367 1368
	tcp_initialize_rcv_mss(newsk);

1369 1370
#ifdef CONFIG_TCP_MD5SIG
	/* Copy over the MD5 key from the original socket */
E
Eric Dumazet 已提交
1371 1372
	key = tcp_md5_do_lookup(sk, (union tcp_md5_addr *)&newinet->inet_daddr,
				AF_INET);
E
Eric Dumazet 已提交
1373
	if (key != NULL) {
1374 1375 1376 1377 1378 1379
		/*
		 * We're using one, so create a matching key
		 * on the newsk structure. If we fail to get
		 * memory, then we end up not copying the key
		 * across. Shucks.
		 */
E
Eric Dumazet 已提交
1380 1381
		tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newinet->inet_daddr,
			       AF_INET, key->key, key->keylen, GFP_ATOMIC);
E
Eric Dumazet 已提交
1382
		sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
1383 1384 1385
	}
#endif

1386 1387
	if (__inet_inherit_port(sk, newsk) < 0)
		goto put_and_exit;
1388
	__inet_hash_nolisten(newsk, NULL);
L
Linus Torvalds 已提交
1389 1390 1391 1392

	return newsk;

exit_overflow:
1393
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1394 1395
exit_nonewsk:
	dst_release(dst);
L
Linus Torvalds 已提交
1396
exit:
1397
	NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_LISTENDROPS);
L
Linus Torvalds 已提交
1398
	return NULL;
1399
put_and_exit:
1400 1401
	inet_csk_prepare_forced_close(newsk);
	tcp_done(newsk);
1402
	goto exit;
L
Linus Torvalds 已提交
1403
}
E
Eric Dumazet 已提交
1404
EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
L
Linus Torvalds 已提交
1405 1406 1407

static struct sock *tcp_v4_hnd_req(struct sock *sk, struct sk_buff *skb)
{
1408
	struct tcphdr *th = tcp_hdr(skb);
1409
	const struct iphdr *iph = ip_hdr(skb);
L
Linus Torvalds 已提交
1410
	struct sock *nsk;
1411
	struct request_sock **prev;
L
Linus Torvalds 已提交
1412
	/* Find possible connection requests. */
1413 1414
	struct request_sock *req = inet_csk_search_req(sk, &prev, th->source,
						       iph->saddr, iph->daddr);
L
Linus Torvalds 已提交
1415
	if (req)
1416
		return tcp_check_req(sk, skb, req, prev, false);
L
Linus Torvalds 已提交
1417

1418
	nsk = inet_lookup_established(sock_net(sk), &tcp_hashinfo, iph->saddr,
1419
			th->source, iph->daddr, th->dest, inet_iif(skb));
L
Linus Torvalds 已提交
1420 1421 1422 1423 1424 1425

	if (nsk) {
		if (nsk->sk_state != TCP_TIME_WAIT) {
			bh_lock_sock(nsk);
			return nsk;
		}
1426
		inet_twsk_put(inet_twsk(nsk));
L
Linus Torvalds 已提交
1427 1428 1429 1430
		return NULL;
	}

#ifdef CONFIG_SYN_COOKIES
1431
	if (!th->syn)
L
Linus Torvalds 已提交
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446
		sk = cookie_v4_check(sk, skb, &(IPCB(skb)->opt));
#endif
	return sk;
}

/* The socket must have it's spinlock held when we get
 * here.
 *
 * We have a potential double-lock case here, so even when
 * doing backlog processing we use the BH locking scheme.
 * This is because we cannot sleep with the original spinlock
 * held.
 */
int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
{
1447 1448
	struct sock *rsk;

L
Linus Torvalds 已提交
1449
	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1450 1451
		struct dst_entry *dst = sk->sk_rx_dst;

1452
		sock_rps_save_rxhash(sk, skb);
1453
		if (dst) {
E
Eric Dumazet 已提交
1454 1455
			if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
			    dst->ops->check(dst, 0) == NULL) {
1456 1457 1458 1459
				dst_release(dst);
				sk->sk_rx_dst = NULL;
			}
		}
1460
		tcp_rcv_established(sk, skb, tcp_hdr(skb), skb->len);
L
Linus Torvalds 已提交
1461 1462 1463
		return 0;
	}

1464
	if (skb->len < tcp_hdrlen(skb) || tcp_checksum_complete(skb))
L
Linus Torvalds 已提交
1465 1466 1467 1468 1469 1470 1471 1472
		goto csum_err;

	if (sk->sk_state == TCP_LISTEN) {
		struct sock *nsk = tcp_v4_hnd_req(sk, skb);
		if (!nsk)
			goto discard;

		if (nsk != sk) {
1473
			sock_rps_save_rxhash(nsk, skb);
1474 1475
			if (tcp_child_process(sk, nsk, skb)) {
				rsk = nsk;
L
Linus Torvalds 已提交
1476
				goto reset;
1477
			}
L
Linus Torvalds 已提交
1478 1479
			return 0;
		}
1480
	} else
1481
		sock_rps_save_rxhash(sk, skb);
1482

1483
	if (tcp_rcv_state_process(sk, skb, tcp_hdr(skb), skb->len)) {
1484
		rsk = sk;
L
Linus Torvalds 已提交
1485
		goto reset;
1486
	}
L
Linus Torvalds 已提交
1487 1488 1489
	return 0;

reset:
1490
	tcp_v4_send_reset(rsk, skb);
L
Linus Torvalds 已提交
1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
discard:
	kfree_skb(skb);
	/* Be careful here. If this function gets more complicated and
	 * gcc suffers from register pressure on the x86, sk (in %ebx)
	 * might be destroyed here. This current version compiles correctly,
	 * but you have been warned.
	 */
	return 0;

csum_err:
1501
	TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_CSUMERRORS);
1502
	TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_INERRS);
L
Linus Torvalds 已提交
1503 1504
	goto discard;
}
E
Eric Dumazet 已提交
1505
EXPORT_SYMBOL(tcp_v4_do_rcv);
L
Linus Torvalds 已提交
1506

1507
void tcp_v4_early_demux(struct sk_buff *skb)
D
David S. Miller 已提交
1508 1509 1510 1511 1512 1513
{
	const struct iphdr *iph;
	const struct tcphdr *th;
	struct sock *sk;

	if (skb->pkt_type != PACKET_HOST)
1514
		return;
D
David S. Miller 已提交
1515

1516
	if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1517
		return;
D
David S. Miller 已提交
1518 1519

	iph = ip_hdr(skb);
1520
	th = tcp_hdr(skb);
D
David S. Miller 已提交
1521 1522

	if (th->doff < sizeof(struct tcphdr) / 4)
1523
		return;
D
David S. Miller 已提交
1524

1525
	sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
D
David S. Miller 已提交
1526
				       iph->saddr, th->source,
1527
				       iph->daddr, ntohs(th->dest),
E
Eric Dumazet 已提交
1528
				       skb->skb_iif);
D
David S. Miller 已提交
1529 1530 1531 1532 1533
	if (sk) {
		skb->sk = sk;
		skb->destructor = sock_edemux;
		if (sk->sk_state != TCP_TIME_WAIT) {
			struct dst_entry *dst = sk->sk_rx_dst;
E
Eric Dumazet 已提交
1534

D
David S. Miller 已提交
1535 1536
			if (dst)
				dst = dst_check(dst, 0);
1537
			if (dst &&
E
Eric Dumazet 已提交
1538
			    inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
1539
				skb_dst_set_noref(skb, dst);
D
David S. Miller 已提交
1540 1541 1542 1543
		}
	}
}

E
Eric Dumazet 已提交
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561
/* Packet is added to VJ-style prequeue for processing in process
 * context, if a reader task is waiting. Apparently, this exciting
 * idea (VJ's mail "Re: query about TCP header on tcp-ip" of 07 Sep 93)
 * failed somewhere. Latency? Burstiness? Well, at least now we will
 * see, why it failed. 8)8)				  --ANK
 *
 */
bool tcp_prequeue(struct sock *sk, struct sk_buff *skb)
{
	struct tcp_sock *tp = tcp_sk(sk);

	if (sysctl_tcp_low_latency || !tp->ucopy.task)
		return false;

	if (skb->len <= tcp_hdrlen(skb) &&
	    skb_queue_len(&tp->ucopy.prequeue) == 0)
		return false;

1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572
	/* Before escaping RCU protected region, we need to take care of skb
	 * dst. Prequeue is only enabled for established sockets.
	 * For such sockets, we might need the skb dst only to set sk->sk_rx_dst
	 * Instead of doing full sk_rx_dst validity here, let's perform
	 * an optimistic check.
	 */
	if (likely(sk->sk_rx_dst))
		skb_dst_drop(skb);
	else
		skb_dst_force(skb);

E
Eric Dumazet 已提交
1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
	__skb_queue_tail(&tp->ucopy.prequeue, skb);
	tp->ucopy.memory += skb->truesize;
	if (tp->ucopy.memory > sk->sk_rcvbuf) {
		struct sk_buff *skb1;

		BUG_ON(sock_owned_by_user(sk));

		while ((skb1 = __skb_dequeue(&tp->ucopy.prequeue)) != NULL) {
			sk_backlog_rcv(sk, skb1);
			NET_INC_STATS_BH(sock_net(sk),
					 LINUX_MIB_TCPPREQUEUEDROPPED);
		}

		tp->ucopy.memory = 0;
	} else if (skb_queue_len(&tp->ucopy.prequeue) == 1) {
		wake_up_interruptible_sync_poll(sk_sleep(sk),
					   POLLIN | POLLRDNORM | POLLRDBAND);
		if (!inet_csk_ack_scheduled(sk))
			inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
						  (3 * tcp_rto_min(sk)) / 4,
						  TCP_RTO_MAX);
	}
	return true;
}
EXPORT_SYMBOL(tcp_prequeue);

L
Linus Torvalds 已提交
1599 1600 1601 1602 1603 1604
/*
 *	From tcp_input.c
 */

int tcp_v4_rcv(struct sk_buff *skb)
{
1605
	const struct iphdr *iph;
1606
	const struct tcphdr *th;
L
Linus Torvalds 已提交
1607 1608
	struct sock *sk;
	int ret;
1609
	struct net *net = dev_net(skb->dev);
L
Linus Torvalds 已提交
1610 1611 1612 1613 1614

	if (skb->pkt_type != PACKET_HOST)
		goto discard_it;

	/* Count it even if it's bad */
1615
	TCP_INC_STATS_BH(net, TCP_MIB_INSEGS);
L
Linus Torvalds 已提交
1616 1617 1618 1619

	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
		goto discard_it;

1620
	th = tcp_hdr(skb);
L
Linus Torvalds 已提交
1621 1622 1623 1624 1625 1626 1627 1628

	if (th->doff < sizeof(struct tcphdr) / 4)
		goto bad_packet;
	if (!pskb_may_pull(skb, th->doff * 4))
		goto discard_it;

	/* An explanation is required here, I think.
	 * Packet length and doff are validated by header prediction,
S
Stephen Hemminger 已提交
1629
	 * provided case of th->doff==0 is eliminated.
L
Linus Torvalds 已提交
1630
	 * So, we defer the checks. */
1631 1632

	if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1633
		goto csum_error;
L
Linus Torvalds 已提交
1634

1635
	th = tcp_hdr(skb);
1636
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
1637 1638 1639 1640
	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
				    skb->len - th->doff * 4);
	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1641
	TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1642
	TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
L
Linus Torvalds 已提交
1643 1644
	TCP_SKB_CB(skb)->sacked	 = 0;

1645
	sk = __inet_lookup_skb(&tcp_hashinfo, skb, th->source, th->dest);
L
Linus Torvalds 已提交
1646 1647 1648
	if (!sk)
		goto no_tcp_socket;

E
Eric Dumazet 已提交
1649 1650 1651 1652
process:
	if (sk->sk_state == TCP_TIME_WAIT)
		goto do_time_wait;

1653 1654
	if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
		NET_INC_STATS_BH(net, LINUX_MIB_TCPMINTTLDROP);
1655
		goto discard_and_relse;
1656
	}
1657

L
Linus Torvalds 已提交
1658 1659
	if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
		goto discard_and_relse;
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671

#ifdef CONFIG_TCP_MD5SIG
	/*
	 * We really want to reject the packet as early as possible
	 * if:
	 *  o We're expecting an MD5'd packet and this is no MD5 tcp option
	 *  o There is an MD5 option and we're not expecting one
	 */
	if (tcp_v4_inbound_md5_hash(sk, skb))
		goto discard_and_relse;
#endif

1672
	nf_reset(skb);
L
Linus Torvalds 已提交
1673

1674
	if (sk_filter(sk, skb))
L
Linus Torvalds 已提交
1675 1676
		goto discard_and_relse;

1677
	sk_mark_napi_id(sk, skb);
L
Linus Torvalds 已提交
1678 1679
	skb->dev = NULL;

1680
	bh_lock_sock_nested(sk);
L
Linus Torvalds 已提交
1681 1682
	ret = 0;
	if (!sock_owned_by_user(sk)) {
C
Chris Leech 已提交
1683 1684 1685
#ifdef CONFIG_NET_DMA
		struct tcp_sock *tp = tcp_sk(sk);
		if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
1686
			tp->ucopy.dma_chan = net_dma_find_channel();
C
Chris Leech 已提交
1687
		if (tp->ucopy.dma_chan)
L
Linus Torvalds 已提交
1688
			ret = tcp_v4_do_rcv(sk, skb);
C
Chris Leech 已提交
1689 1690 1691 1692
		else
#endif
		{
			if (!tcp_prequeue(sk, skb))
S
Shan Wei 已提交
1693
				ret = tcp_v4_do_rcv(sk, skb);
C
Chris Leech 已提交
1694
		}
1695 1696
	} else if (unlikely(sk_add_backlog(sk, skb,
					   sk->sk_rcvbuf + sk->sk_sndbuf))) {
Z
Zhu Yi 已提交
1697
		bh_unlock_sock(sk);
1698
		NET_INC_STATS_BH(net, LINUX_MIB_TCPBACKLOGDROP);
Z
Zhu Yi 已提交
1699 1700
		goto discard_and_relse;
	}
L
Linus Torvalds 已提交
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
	bh_unlock_sock(sk);

	sock_put(sk);

	return ret;

no_tcp_socket:
	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
		goto discard_it;

	if (skb->len < (th->doff << 2) || tcp_checksum_complete(skb)) {
1712 1713
csum_error:
		TCP_INC_STATS_BH(net, TCP_MIB_CSUMERRORS);
L
Linus Torvalds 已提交
1714
bad_packet:
1715
		TCP_INC_STATS_BH(net, TCP_MIB_INERRS);
L
Linus Torvalds 已提交
1716
	} else {
1717
		tcp_v4_send_reset(NULL, skb);
L
Linus Torvalds 已提交
1718 1719 1720 1721 1722
	}

discard_it:
	/* Discard frame. */
	kfree_skb(skb);
1723
	return 0;
L
Linus Torvalds 已提交
1724 1725 1726 1727 1728 1729 1730

discard_and_relse:
	sock_put(sk);
	goto discard_it;

do_time_wait:
	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1731
		inet_twsk_put(inet_twsk(sk));
L
Linus Torvalds 已提交
1732 1733 1734
		goto discard_it;
	}

1735
	if (skb->len < (th->doff << 2)) {
1736
		inet_twsk_put(inet_twsk(sk));
1737 1738 1739 1740 1741
		goto bad_packet;
	}
	if (tcp_checksum_complete(skb)) {
		inet_twsk_put(inet_twsk(sk));
		goto csum_error;
L
Linus Torvalds 已提交
1742
	}
1743
	switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
L
Linus Torvalds 已提交
1744
	case TCP_TW_SYN: {
1745
		struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
1746
							&tcp_hashinfo,
1747
							iph->saddr, th->source,
1748
							iph->daddr, th->dest,
1749
							inet_iif(skb));
L
Linus Torvalds 已提交
1750
		if (sk2) {
1751 1752
			inet_twsk_deschedule(inet_twsk(sk), &tcp_death_row);
			inet_twsk_put(inet_twsk(sk));
L
Linus Torvalds 已提交
1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
			sk = sk2;
			goto process;
		}
		/* Fall through to ACK */
	}
	case TCP_TW_ACK:
		tcp_v4_timewait_ack(sk, skb);
		break;
	case TCP_TW_RST:
		goto no_tcp_socket;
	case TCP_TW_SUCCESS:;
	}
	goto discard_it;
}

1768 1769 1770 1771 1772
static struct timewait_sock_ops tcp_timewait_sock_ops = {
	.twsk_obj_size	= sizeof(struct tcp_timewait_sock),
	.twsk_unique	= tcp_twsk_unique,
	.twsk_destructor= tcp_twsk_destructor,
};
L
Linus Torvalds 已提交
1773

1774
void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
E
Eric Dumazet 已提交
1775 1776 1777
{
	struct dst_entry *dst = skb_dst(skb);

1778 1779 1780 1781 1782
	if (dst) {
		dst_hold(dst);
		sk->sk_rx_dst = dst;
		inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
	}
E
Eric Dumazet 已提交
1783
}
1784
EXPORT_SYMBOL(inet_sk_rx_dst_set);
E
Eric Dumazet 已提交
1785

1786
const struct inet_connection_sock_af_ops ipv4_specific = {
1787 1788 1789
	.queue_xmit	   = ip_queue_xmit,
	.send_check	   = tcp_v4_send_check,
	.rebuild_header	   = inet_sk_rebuild_header,
E
Eric Dumazet 已提交
1790
	.sk_rx_dst_set	   = inet_sk_rx_dst_set,
1791 1792 1793 1794 1795 1796 1797
	.conn_request	   = tcp_v4_conn_request,
	.syn_recv_sock	   = tcp_v4_syn_recv_sock,
	.net_header_len	   = sizeof(struct iphdr),
	.setsockopt	   = ip_setsockopt,
	.getsockopt	   = ip_getsockopt,
	.addr2sockaddr	   = inet_csk_addr2sockaddr,
	.sockaddr_len	   = sizeof(struct sockaddr_in),
1798
	.bind_conflict	   = inet_csk_bind_conflict,
1799
#ifdef CONFIG_COMPAT
1800 1801
	.compat_setsockopt = compat_ip_setsockopt,
	.compat_getsockopt = compat_ip_getsockopt,
1802
#endif
1803
	.mtu_reduced	   = tcp_v4_mtu_reduced,
L
Linus Torvalds 已提交
1804
};
E
Eric Dumazet 已提交
1805
EXPORT_SYMBOL(ipv4_specific);
L
Linus Torvalds 已提交
1806

1807
#ifdef CONFIG_TCP_MD5SIG
1808
static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
1809
	.md5_lookup		= tcp_v4_md5_lookup,
1810
	.calc_md5_hash		= tcp_v4_md5_hash_skb,
1811 1812
	.md5_parse		= tcp_v4_parse_md5_keys,
};
1813
#endif
1814

L
Linus Torvalds 已提交
1815 1816 1817 1818 1819
/* NOTE: A lot of things set to zero explicitly by call to
 *       sk_alloc() so need not be done here.
 */
static int tcp_v4_init_sock(struct sock *sk)
{
1820
	struct inet_connection_sock *icsk = inet_csk(sk);
L
Linus Torvalds 已提交
1821

1822
	tcp_init_sock(sk);
L
Linus Torvalds 已提交
1823

1824
	icsk->icsk_af_ops = &ipv4_specific;
1825

1826
#ifdef CONFIG_TCP_MD5SIG
1827
	tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
1828
#endif
L
Linus Torvalds 已提交
1829 1830 1831 1832

	return 0;
}

1833
void tcp_v4_destroy_sock(struct sock *sk)
L
Linus Torvalds 已提交
1834 1835 1836 1837 1838
{
	struct tcp_sock *tp = tcp_sk(sk);

	tcp_clear_xmit_timers(sk);

1839
	tcp_cleanup_congestion_control(sk);
1840

L
Linus Torvalds 已提交
1841
	/* Cleanup up the write buffer. */
1842
	tcp_write_queue_purge(sk);
L
Linus Torvalds 已提交
1843 1844

	/* Cleans up our, hopefully empty, out_of_order_queue. */
1845
	__skb_queue_purge(&tp->out_of_order_queue);
L
Linus Torvalds 已提交
1846

1847 1848 1849
#ifdef CONFIG_TCP_MD5SIG
	/* Clean up the MD5 key list, if any */
	if (tp->md5sig_info) {
E
Eric Dumazet 已提交
1850
		tcp_clear_md5_list(sk);
1851
		kfree_rcu(tp->md5sig_info, rcu);
1852 1853 1854 1855
		tp->md5sig_info = NULL;
	}
#endif

C
Chris Leech 已提交
1856 1857
#ifdef CONFIG_NET_DMA
	/* Cleans up our sk_async_wait_queue */
1858
	__skb_queue_purge(&sk->sk_async_wait_queue);
C
Chris Leech 已提交
1859 1860
#endif

L
Linus Torvalds 已提交
1861 1862 1863 1864
	/* Clean prequeue, it must be empty really */
	__skb_queue_purge(&tp->ucopy.prequeue);

	/* Clean up a referenced TCP bind bucket. */
1865
	if (inet_csk(sk)->icsk_bind_hash)
1866
		inet_put_port(sk);
L
Linus Torvalds 已提交
1867

1868
	BUG_ON(tp->fastopen_rsk != NULL);
1869

1870 1871 1872
	/* If socket is aborted during connect operation */
	tcp_free_fastopen_req(tp);

1873
	sk_sockets_allocated_dec(sk);
G
Glauber Costa 已提交
1874
	sock_release_memcg(sk);
L
Linus Torvalds 已提交
1875 1876 1877 1878 1879 1880
}
EXPORT_SYMBOL(tcp_v4_destroy_sock);

#ifdef CONFIG_PROC_FS
/* Proc filesystem TCP sock list dumping. */

1881 1882 1883 1884 1885
/*
 * Get next listener socket follow cur.  If cur is NULL, get first socket
 * starting from bucket given in st->bucket; when st->bucket is zero the
 * very first socket in the hash table is returned.
 */
L
Linus Torvalds 已提交
1886 1887
static void *listening_get_next(struct seq_file *seq, void *cur)
{
1888
	struct inet_connection_sock *icsk;
1889
	struct hlist_nulls_node *node;
L
Linus Torvalds 已提交
1890
	struct sock *sk = cur;
1891
	struct inet_listen_hashbucket *ilb;
J
Jianjun Kong 已提交
1892
	struct tcp_iter_state *st = seq->private;
1893
	struct net *net = seq_file_net(seq);
L
Linus Torvalds 已提交
1894 1895

	if (!sk) {
1896
		ilb = &tcp_hashinfo.listening_hash[st->bucket];
1897
		spin_lock_bh(&ilb->lock);
1898
		sk = sk_nulls_head(&ilb->head);
1899
		st->offset = 0;
L
Linus Torvalds 已提交
1900 1901
		goto get_sk;
	}
1902
	ilb = &tcp_hashinfo.listening_hash[st->bucket];
L
Linus Torvalds 已提交
1903
	++st->num;
1904
	++st->offset;
L
Linus Torvalds 已提交
1905 1906

	if (st->state == TCP_SEQ_STATE_OPENREQ) {
1907
		struct request_sock *req = cur;
L
Linus Torvalds 已提交
1908

1909
		icsk = inet_csk(st->syn_wait_sk);
L
Linus Torvalds 已提交
1910 1911 1912
		req = req->dl_next;
		while (1) {
			while (req) {
1913
				if (req->rsk_ops->family == st->family) {
L
Linus Torvalds 已提交
1914 1915 1916 1917 1918
					cur = req;
					goto out;
				}
				req = req->dl_next;
			}
1919
			if (++st->sbucket >= icsk->icsk_accept_queue.listen_opt->nr_table_entries)
L
Linus Torvalds 已提交
1920 1921
				break;
get_req:
1922
			req = icsk->icsk_accept_queue.listen_opt->syn_table[st->sbucket];
L
Linus Torvalds 已提交
1923
		}
E
Eric Dumazet 已提交
1924
		sk	  = sk_nulls_next(st->syn_wait_sk);
L
Linus Torvalds 已提交
1925
		st->state = TCP_SEQ_STATE_LISTENING;
1926
		read_unlock_bh(&icsk->icsk_accept_queue.syn_wait_lock);
L
Linus Torvalds 已提交
1927
	} else {
1928
		icsk = inet_csk(sk);
1929 1930
		read_lock_bh(&icsk->icsk_accept_queue.syn_wait_lock);
		if (reqsk_queue_len(&icsk->icsk_accept_queue))
L
Linus Torvalds 已提交
1931
			goto start_req;
1932
		read_unlock_bh(&icsk->icsk_accept_queue.syn_wait_lock);
E
Eric Dumazet 已提交
1933
		sk = sk_nulls_next(sk);
L
Linus Torvalds 已提交
1934 1935
	}
get_sk:
1936
	sk_nulls_for_each_from(sk, node) {
1937 1938 1939
		if (!net_eq(sock_net(sk), net))
			continue;
		if (sk->sk_family == st->family) {
L
Linus Torvalds 已提交
1940 1941 1942
			cur = sk;
			goto out;
		}
1943
		icsk = inet_csk(sk);
1944 1945
		read_lock_bh(&icsk->icsk_accept_queue.syn_wait_lock);
		if (reqsk_queue_len(&icsk->icsk_accept_queue)) {
L
Linus Torvalds 已提交
1946 1947 1948 1949 1950 1951 1952
start_req:
			st->uid		= sock_i_uid(sk);
			st->syn_wait_sk = sk;
			st->state	= TCP_SEQ_STATE_OPENREQ;
			st->sbucket	= 0;
			goto get_req;
		}
1953
		read_unlock_bh(&icsk->icsk_accept_queue.syn_wait_lock);
L
Linus Torvalds 已提交
1954
	}
1955
	spin_unlock_bh(&ilb->lock);
1956
	st->offset = 0;
1957
	if (++st->bucket < INET_LHTABLE_SIZE) {
1958 1959
		ilb = &tcp_hashinfo.listening_hash[st->bucket];
		spin_lock_bh(&ilb->lock);
1960
		sk = sk_nulls_head(&ilb->head);
L
Linus Torvalds 已提交
1961 1962 1963 1964 1965 1966 1967 1968 1969
		goto get_sk;
	}
	cur = NULL;
out:
	return cur;
}

static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
{
1970 1971 1972 1973 1974 1975
	struct tcp_iter_state *st = seq->private;
	void *rc;

	st->bucket = 0;
	st->offset = 0;
	rc = listening_get_next(seq, NULL);
L
Linus Torvalds 已提交
1976 1977 1978 1979 1980 1981 1982 1983

	while (rc && *pos) {
		rc = listening_get_next(seq, rc);
		--*pos;
	}
	return rc;
}

E
Eric Dumazet 已提交
1984
static inline bool empty_bucket(const struct tcp_iter_state *st)
1985
{
E
Eric Dumazet 已提交
1986
	return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
1987 1988
}

1989 1990 1991 1992
/*
 * Get first established socket starting from bucket given in st->bucket.
 * If st->bucket is zero, the very first socket in the hash is returned.
 */
L
Linus Torvalds 已提交
1993 1994
static void *established_get_first(struct seq_file *seq)
{
J
Jianjun Kong 已提交
1995
	struct tcp_iter_state *st = seq->private;
1996
	struct net *net = seq_file_net(seq);
L
Linus Torvalds 已提交
1997 1998
	void *rc = NULL;

1999 2000
	st->offset = 0;
	for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
L
Linus Torvalds 已提交
2001
		struct sock *sk;
2002
		struct hlist_nulls_node *node;
2003
		spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
L
Linus Torvalds 已提交
2004

2005 2006 2007 2008
		/* Lockless fast path for the common case of empty buckets */
		if (empty_bucket(st))
			continue;

2009
		spin_lock_bh(lock);
2010
		sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
2011
			if (sk->sk_family != st->family ||
2012
			    !net_eq(sock_net(sk), net)) {
L
Linus Torvalds 已提交
2013 2014 2015 2016 2017
				continue;
			}
			rc = sk;
			goto out;
		}
2018
		spin_unlock_bh(lock);
L
Linus Torvalds 已提交
2019 2020 2021 2022 2023 2024 2025 2026
	}
out:
	return rc;
}

static void *established_get_next(struct seq_file *seq, void *cur)
{
	struct sock *sk = cur;
2027
	struct hlist_nulls_node *node;
J
Jianjun Kong 已提交
2028
	struct tcp_iter_state *st = seq->private;
2029
	struct net *net = seq_file_net(seq);
L
Linus Torvalds 已提交
2030 2031

	++st->num;
2032
	++st->offset;
L
Linus Torvalds 已提交
2033

E
Eric Dumazet 已提交
2034
	sk = sk_nulls_next(sk);
L
Linus Torvalds 已提交
2035

2036
	sk_nulls_for_each_from(sk, node) {
2037
		if (sk->sk_family == st->family && net_eq(sock_net(sk), net))
E
Eric Dumazet 已提交
2038
			return sk;
L
Linus Torvalds 已提交
2039 2040
	}

E
Eric Dumazet 已提交
2041 2042 2043
	spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
	++st->bucket;
	return established_get_first(seq);
L
Linus Torvalds 已提交
2044 2045 2046 2047
}

static void *established_get_idx(struct seq_file *seq, loff_t pos)
{
2048 2049 2050 2051 2052
	struct tcp_iter_state *st = seq->private;
	void *rc;

	st->bucket = 0;
	rc = established_get_first(seq);
L
Linus Torvalds 已提交
2053 2054 2055 2056

	while (rc && pos) {
		rc = established_get_next(seq, rc);
		--pos;
2057
	}
L
Linus Torvalds 已提交
2058 2059 2060 2061 2062 2063
	return rc;
}

static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
{
	void *rc;
J
Jianjun Kong 已提交
2064
	struct tcp_iter_state *st = seq->private;
L
Linus Torvalds 已提交
2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076

	st->state = TCP_SEQ_STATE_LISTENING;
	rc	  = listening_get_idx(seq, &pos);

	if (!rc) {
		st->state = TCP_SEQ_STATE_ESTABLISHED;
		rc	  = established_get_idx(seq, pos);
	}

	return rc;
}

2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
static void *tcp_seek_last_pos(struct seq_file *seq)
{
	struct tcp_iter_state *st = seq->private;
	int offset = st->offset;
	int orig_num = st->num;
	void *rc = NULL;

	switch (st->state) {
	case TCP_SEQ_STATE_OPENREQ:
	case TCP_SEQ_STATE_LISTENING:
		if (st->bucket >= INET_LHTABLE_SIZE)
			break;
		st->state = TCP_SEQ_STATE_LISTENING;
		rc = listening_get_next(seq, NULL);
		while (offset-- && rc)
			rc = listening_get_next(seq, rc);
		if (rc)
			break;
		st->bucket = 0;
E
Eric Dumazet 已提交
2096
		st->state = TCP_SEQ_STATE_ESTABLISHED;
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110
		/* Fallthrough */
	case TCP_SEQ_STATE_ESTABLISHED:
		if (st->bucket > tcp_hashinfo.ehash_mask)
			break;
		rc = established_get_first(seq);
		while (offset-- && rc)
			rc = established_get_next(seq, rc);
	}

	st->num = orig_num;

	return rc;
}

L
Linus Torvalds 已提交
2111 2112
static void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
{
J
Jianjun Kong 已提交
2113
	struct tcp_iter_state *st = seq->private;
2114 2115 2116 2117 2118 2119 2120 2121
	void *rc;

	if (*pos && *pos == st->last_pos) {
		rc = tcp_seek_last_pos(seq);
		if (rc)
			goto out;
	}

L
Linus Torvalds 已提交
2122 2123
	st->state = TCP_SEQ_STATE_LISTENING;
	st->num = 0;
2124 2125 2126 2127 2128 2129 2130
	st->bucket = 0;
	st->offset = 0;
	rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;

out:
	st->last_pos = *pos;
	return rc;
L
Linus Torvalds 已提交
2131 2132 2133 2134
}

static void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
2135
	struct tcp_iter_state *st = seq->private;
L
Linus Torvalds 已提交
2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
	void *rc = NULL;

	if (v == SEQ_START_TOKEN) {
		rc = tcp_get_idx(seq, 0);
		goto out;
	}

	switch (st->state) {
	case TCP_SEQ_STATE_OPENREQ:
	case TCP_SEQ_STATE_LISTENING:
		rc = listening_get_next(seq, v);
		if (!rc) {
			st->state = TCP_SEQ_STATE_ESTABLISHED;
2149 2150
			st->bucket = 0;
			st->offset = 0;
L
Linus Torvalds 已提交
2151 2152 2153 2154 2155 2156 2157 2158 2159
			rc	  = established_get_first(seq);
		}
		break;
	case TCP_SEQ_STATE_ESTABLISHED:
		rc = established_get_next(seq, v);
		break;
	}
out:
	++*pos;
2160
	st->last_pos = *pos;
L
Linus Torvalds 已提交
2161 2162 2163 2164 2165
	return rc;
}

static void tcp_seq_stop(struct seq_file *seq, void *v)
{
J
Jianjun Kong 已提交
2166
	struct tcp_iter_state *st = seq->private;
L
Linus Torvalds 已提交
2167 2168 2169 2170

	switch (st->state) {
	case TCP_SEQ_STATE_OPENREQ:
		if (v) {
2171 2172
			struct inet_connection_sock *icsk = inet_csk(st->syn_wait_sk);
			read_unlock_bh(&icsk->icsk_accept_queue.syn_wait_lock);
L
Linus Torvalds 已提交
2173 2174 2175
		}
	case TCP_SEQ_STATE_LISTENING:
		if (v != SEQ_START_TOKEN)
2176
			spin_unlock_bh(&tcp_hashinfo.listening_hash[st->bucket].lock);
L
Linus Torvalds 已提交
2177 2178 2179
		break;
	case TCP_SEQ_STATE_ESTABLISHED:
		if (v)
2180
			spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
L
Linus Torvalds 已提交
2181 2182 2183 2184
		break;
	}
}

2185
int tcp_seq_open(struct inode *inode, struct file *file)
L
Linus Torvalds 已提交
2186
{
A
Al Viro 已提交
2187
	struct tcp_seq_afinfo *afinfo = PDE_DATA(inode);
L
Linus Torvalds 已提交
2188
	struct tcp_iter_state *s;
2189
	int err;
L
Linus Torvalds 已提交
2190

2191 2192 2193 2194
	err = seq_open_net(inode, file, &afinfo->seq_ops,
			  sizeof(struct tcp_iter_state));
	if (err < 0)
		return err;
2195

2196
	s = ((struct seq_file *)file->private_data)->private;
L
Linus Torvalds 已提交
2197
	s->family		= afinfo->family;
S
stephen hemminger 已提交
2198
	s->last_pos		= 0;
2199 2200
	return 0;
}
2201
EXPORT_SYMBOL(tcp_seq_open);
2202

2203
int tcp_proc_register(struct net *net, struct tcp_seq_afinfo *afinfo)
L
Linus Torvalds 已提交
2204 2205 2206 2207
{
	int rc = 0;
	struct proc_dir_entry *p;

2208 2209 2210 2211
	afinfo->seq_ops.start		= tcp_seq_start;
	afinfo->seq_ops.next		= tcp_seq_next;
	afinfo->seq_ops.stop		= tcp_seq_stop;

2212
	p = proc_create_data(afinfo->name, S_IRUGO, net->proc_net,
2213
			     afinfo->seq_fops, afinfo);
2214
	if (!p)
L
Linus Torvalds 已提交
2215 2216 2217
		rc = -ENOMEM;
	return rc;
}
E
Eric Dumazet 已提交
2218
EXPORT_SYMBOL(tcp_proc_register);
L
Linus Torvalds 已提交
2219

2220
void tcp_proc_unregister(struct net *net, struct tcp_seq_afinfo *afinfo)
L
Linus Torvalds 已提交
2221
{
2222
	remove_proc_entry(afinfo->name, net->proc_net);
L
Linus Torvalds 已提交
2223
}
E
Eric Dumazet 已提交
2224
EXPORT_SYMBOL(tcp_proc_unregister);
L
Linus Torvalds 已提交
2225

2226
static void get_openreq4(const struct sock *sk, const struct request_sock *req,
2227
			 struct seq_file *f, int i, kuid_t uid)
L
Linus Torvalds 已提交
2228
{
2229
	const struct inet_request_sock *ireq = inet_rsk(req);
2230
	long delta = req->expires - jiffies;
L
Linus Torvalds 已提交
2231

2232
	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2233
		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
L
Linus Torvalds 已提交
2234
		i,
2235
		ireq->ir_loc_addr,
E
Eric Dumazet 已提交
2236
		ntohs(inet_sk(sk)->inet_sport),
2237 2238
		ireq->ir_rmt_addr,
		ntohs(ireq->ir_rmt_port),
L
Linus Torvalds 已提交
2239 2240 2241
		TCP_SYN_RECV,
		0, 0, /* could print option size, but that is af dependent. */
		1,    /* timers active (only the expire timer) */
2242
		jiffies_delta_to_clock_t(delta),
2243
		req->num_timeout,
2244
		from_kuid_munged(seq_user_ns(f), uid),
L
Linus Torvalds 已提交
2245 2246 2247
		0,  /* non standard timer */
		0, /* open_requests have no inode */
		atomic_read(&sk->sk_refcnt),
2248
		req);
L
Linus Torvalds 已提交
2249 2250
}

2251
static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
L
Linus Torvalds 已提交
2252 2253 2254
{
	int timer_active;
	unsigned long timer_expires;
2255
	const struct tcp_sock *tp = tcp_sk(sk);
2256
	const struct inet_connection_sock *icsk = inet_csk(sk);
2257
	const struct inet_sock *inet = inet_sk(sk);
2258
	struct fastopen_queue *fastopenq = icsk->icsk_accept_queue.fastopenq;
E
Eric Dumazet 已提交
2259 2260 2261 2262
	__be32 dest = inet->inet_daddr;
	__be32 src = inet->inet_rcv_saddr;
	__u16 destp = ntohs(inet->inet_dport);
	__u16 srcp = ntohs(inet->inet_sport);
2263
	int rx_queue;
L
Linus Torvalds 已提交
2264

N
Nandita Dukkipati 已提交
2265 2266 2267
	if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
	    icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS ||
	    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
L
Linus Torvalds 已提交
2268
		timer_active	= 1;
2269 2270
		timer_expires	= icsk->icsk_timeout;
	} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
L
Linus Torvalds 已提交
2271
		timer_active	= 4;
2272
		timer_expires	= icsk->icsk_timeout;
2273
	} else if (timer_pending(&sk->sk_timer)) {
L
Linus Torvalds 已提交
2274
		timer_active	= 2;
2275
		timer_expires	= sk->sk_timer.expires;
L
Linus Torvalds 已提交
2276 2277 2278 2279 2280
	} else {
		timer_active	= 0;
		timer_expires = jiffies;
	}

2281 2282 2283 2284 2285 2286 2287 2288
	if (sk->sk_state == TCP_LISTEN)
		rx_queue = sk->sk_ack_backlog;
	else
		/*
		 * because we dont lock socket, we might find a transient negative value
		 */
		rx_queue = max_t(int, tp->rcv_nxt - tp->copied_seq, 0);

2289
	seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2290
			"%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2291
		i, src, srcp, dest, destp, sk->sk_state,
2292
		tp->write_seq - tp->snd_una,
2293
		rx_queue,
L
Linus Torvalds 已提交
2294
		timer_active,
2295
		jiffies_delta_to_clock_t(timer_expires - jiffies),
2296
		icsk->icsk_retransmits,
2297
		from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2298
		icsk->icsk_probes_out,
2299 2300
		sock_i_ino(sk),
		atomic_read(&sk->sk_refcnt), sk,
2301 2302
		jiffies_to_clock_t(icsk->icsk_rto),
		jiffies_to_clock_t(icsk->icsk_ack.ato),
2303
		(icsk->icsk_ack.quick << 1) | icsk->icsk_ack.pingpong,
L
Linus Torvalds 已提交
2304
		tp->snd_cwnd,
2305 2306
		sk->sk_state == TCP_LISTEN ?
		    (fastopenq ? fastopenq->max_qlen : 0) :
2307
		    (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
L
Linus Torvalds 已提交
2308 2309
}

2310
static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2311
			       struct seq_file *f, int i)
L
Linus Torvalds 已提交
2312
{
2313
	__be32 dest, src;
L
Linus Torvalds 已提交
2314
	__u16 destp, srcp;
2315
	s32 delta = tw->tw_ttd - inet_tw_time_stamp();
L
Linus Torvalds 已提交
2316 2317 2318 2319 2320 2321

	dest  = tw->tw_daddr;
	src   = tw->tw_rcv_saddr;
	destp = ntohs(tw->tw_dport);
	srcp  = ntohs(tw->tw_sport);

2322
	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2323
		" %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
L
Linus Torvalds 已提交
2324
		i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2325
		3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2326
		atomic_read(&tw->tw_refcnt), tw);
L
Linus Torvalds 已提交
2327 2328 2329 2330 2331 2332
}

#define TMPSZ 150

static int tcp4_seq_show(struct seq_file *seq, void *v)
{
J
Jianjun Kong 已提交
2333
	struct tcp_iter_state *st;
E
Eric Dumazet 已提交
2334
	struct sock *sk = v;
L
Linus Torvalds 已提交
2335

2336
	seq_setwidth(seq, TMPSZ - 1);
L
Linus Torvalds 已提交
2337
	if (v == SEQ_START_TOKEN) {
2338
		seq_puts(seq, "  sl  local_address rem_address   st tx_queue "
L
Linus Torvalds 已提交
2339 2340 2341 2342 2343 2344 2345 2346 2347
			   "rx_queue tr tm->when retrnsmt   uid  timeout "
			   "inode");
		goto out;
	}
	st = seq->private;

	switch (st->state) {
	case TCP_SEQ_STATE_LISTENING:
	case TCP_SEQ_STATE_ESTABLISHED:
E
Eric Dumazet 已提交
2348
		if (sk->sk_state == TCP_TIME_WAIT)
2349
			get_timewait4_sock(v, seq, st->num);
E
Eric Dumazet 已提交
2350
		else
2351
			get_tcp4_sock(v, seq, st->num);
L
Linus Torvalds 已提交
2352 2353
		break;
	case TCP_SEQ_STATE_OPENREQ:
2354
		get_openreq4(st->syn_wait_sk, v, seq, st->num, st->uid);
L
Linus Torvalds 已提交
2355 2356 2357
		break;
	}
out:
2358
	seq_pad(seq, '\n');
L
Linus Torvalds 已提交
2359 2360 2361
	return 0;
}

2362 2363 2364 2365 2366 2367 2368 2369
static const struct file_operations tcp_afinfo_seq_fops = {
	.owner   = THIS_MODULE,
	.open    = tcp_seq_open,
	.read    = seq_read,
	.llseek  = seq_lseek,
	.release = seq_release_net
};

L
Linus Torvalds 已提交
2370 2371 2372
static struct tcp_seq_afinfo tcp4_seq_afinfo = {
	.name		= "tcp",
	.family		= AF_INET,
2373
	.seq_fops	= &tcp_afinfo_seq_fops,
2374 2375 2376
	.seq_ops	= {
		.show		= tcp4_seq_show,
	},
L
Linus Torvalds 已提交
2377 2378
};

2379
static int __net_init tcp4_proc_init_net(struct net *net)
2380 2381 2382 2383
{
	return tcp_proc_register(net, &tcp4_seq_afinfo);
}

2384
static void __net_exit tcp4_proc_exit_net(struct net *net)
2385 2386 2387 2388 2389 2390 2391 2392 2393
{
	tcp_proc_unregister(net, &tcp4_seq_afinfo);
}

static struct pernet_operations tcp4_net_ops = {
	.init = tcp4_proc_init_net,
	.exit = tcp4_proc_exit_net,
};

L
Linus Torvalds 已提交
2394 2395
int __init tcp4_proc_init(void)
{
2396
	return register_pernet_subsys(&tcp4_net_ops);
L
Linus Torvalds 已提交
2397 2398 2399 2400
}

void tcp4_proc_exit(void)
{
2401
	unregister_pernet_subsys(&tcp4_net_ops);
L
Linus Torvalds 已提交
2402 2403 2404 2405 2406 2407 2408 2409 2410
}
#endif /* CONFIG_PROC_FS */

struct proto tcp_prot = {
	.name			= "TCP",
	.owner			= THIS_MODULE,
	.close			= tcp_close,
	.connect		= tcp_v4_connect,
	.disconnect		= tcp_disconnect,
2411
	.accept			= inet_csk_accept,
L
Linus Torvalds 已提交
2412 2413 2414 2415 2416 2417 2418
	.ioctl			= tcp_ioctl,
	.init			= tcp_v4_init_sock,
	.destroy		= tcp_v4_destroy_sock,
	.shutdown		= tcp_shutdown,
	.setsockopt		= tcp_setsockopt,
	.getsockopt		= tcp_getsockopt,
	.recvmsg		= tcp_recvmsg,
2419 2420
	.sendmsg		= tcp_sendmsg,
	.sendpage		= tcp_sendpage,
L
Linus Torvalds 已提交
2421
	.backlog_rcv		= tcp_v4_do_rcv,
E
Eric Dumazet 已提交
2422
	.release_cb		= tcp_release_cb,
2423 2424 2425
	.hash			= inet_hash,
	.unhash			= inet_unhash,
	.get_port		= inet_csk_get_port,
L
Linus Torvalds 已提交
2426
	.enter_memory_pressure	= tcp_enter_memory_pressure,
2427
	.stream_memory_free	= tcp_stream_memory_free,
L
Linus Torvalds 已提交
2428
	.sockets_allocated	= &tcp_sockets_allocated,
2429
	.orphan_count		= &tcp_orphan_count,
L
Linus Torvalds 已提交
2430 2431
	.memory_allocated	= &tcp_memory_allocated,
	.memory_pressure	= &tcp_memory_pressure,
2432
	.sysctl_mem		= sysctl_tcp_mem,
L
Linus Torvalds 已提交
2433 2434 2435 2436
	.sysctl_wmem		= sysctl_tcp_wmem,
	.sysctl_rmem		= sysctl_tcp_rmem,
	.max_header		= MAX_TCP_HEADER,
	.obj_size		= sizeof(struct tcp_sock),
2437
	.slab_flags		= SLAB_DESTROY_BY_RCU,
2438
	.twsk_prot		= &tcp_timewait_sock_ops,
2439
	.rsk_prot		= &tcp_request_sock_ops,
2440
	.h.hashinfo		= &tcp_hashinfo,
2441
	.no_autobind		= true,
2442 2443 2444 2445
#ifdef CONFIG_COMPAT
	.compat_setsockopt	= compat_tcp_setsockopt,
	.compat_getsockopt	= compat_tcp_getsockopt,
#endif
A
Andrew Morton 已提交
2446
#ifdef CONFIG_MEMCG_KMEM
G
Glauber Costa 已提交
2447 2448 2449 2450
	.init_cgroup		= tcp_init_cgroup,
	.destroy_cgroup		= tcp_destroy_cgroup,
	.proto_cgroup		= tcp_proto_cgroup,
#endif
L
Linus Torvalds 已提交
2451
};
E
Eric Dumazet 已提交
2452
EXPORT_SYMBOL(tcp_prot);
L
Linus Torvalds 已提交
2453

2454 2455
static int __net_init tcp_sk_init(struct net *net)
{
2456
	net->ipv4.sysctl_tcp_ecn = 2;
2457
	return 0;
2458 2459 2460 2461
}

static void __net_exit tcp_sk_exit(struct net *net)
{
E
Eric W. Biederman 已提交
2462 2463 2464 2465 2466
}

static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
{
	inet_twsk_purge(&tcp_hashinfo, &tcp_death_row, AF_INET);
2467 2468 2469
}

static struct pernet_operations __net_initdata tcp_sk_ops = {
E
Eric W. Biederman 已提交
2470 2471 2472
       .init	   = tcp_sk_init,
       .exit	   = tcp_sk_exit,
       .exit_batch = tcp_sk_exit_batch,
2473 2474
};

2475
void __init tcp_v4_init(void)
L
Linus Torvalds 已提交
2476
{
2477
	inet_hashinfo_init(&tcp_hashinfo);
2478
	if (register_pernet_subsys(&tcp_sk_ops))
L
Linus Torvalds 已提交
2479 2480
		panic("Failed to create the TCP control socket.\n");
}