protocol.c 84.0 KB
Newer Older
M
Mat Martineau 已提交
1 2 3 4 5 6 7 8 9 10 11
// SPDX-License-Identifier: GPL-2.0
/* Multipath TCP
 *
 * Copyright (c) 2017 - 2019, Intel Corporation.
 */

#define pr_fmt(fmt) "MPTCP: " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/netdevice.h>
12 13
#include <linux/sched/signal.h>
#include <linux/atomic.h>
M
Mat Martineau 已提交
14 15 16 17 18
#include <net/sock.h>
#include <net/inet_common.h>
#include <net/inet_hashtables.h>
#include <net/protocol.h>
#include <net/tcp.h>
19
#include <net/tcp_states.h>
20 21 22
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
#include <net/transp_v6.h>
#endif
M
Mat Martineau 已提交
23
#include <net/mptcp.h>
P
Paolo Abeni 已提交
24
#include <net/xfrm.h>
M
Mat Martineau 已提交
25
#include "protocol.h"
26
#include "mib.h"
M
Mat Martineau 已提交
27

28 29 30 31 32 33 34
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
struct mptcp6_sock {
	struct mptcp_sock msk;
	struct ipv6_pinfo np;
};
#endif

35
struct mptcp_skb_cb {
36 37
	u64 map_seq;
	u64 end_seq;
38 39 40 41 42
	u32 offset;
};

#define MPTCP_SKB_CB(__skb)	((struct mptcp_skb_cb *)&((__skb)->cb[0]))

43 44
static struct percpu_counter mptcp_sockets_allocated;

P
Paolo Abeni 已提交
45
static void __mptcp_destroy_sock(struct sock *sk);
46
static void __mptcp_check_send_data_fin(struct sock *sk);
P
Paolo Abeni 已提交
47

48 49 50 51 52 53
/* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not
 * completed yet or has failed, return the subflow socket.
 * Otherwise return NULL.
 */
static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk)
{
54
	if (!msk->subflow || READ_ONCE(msk->can_ack))
55 56 57 58 59
		return NULL;

	return msk->subflow;
}

60 61 62
/* Returns end sequence number of the receiver's advertised window */
static u64 mptcp_wnd_end(const struct mptcp_sock *msk)
{
63
	return READ_ONCE(msk->wnd_end);
64 65
}

66
static bool mptcp_is_tcpsk(struct sock *sk)
F
Florian Westphal 已提交
67 68 69 70 71 72 73 74 75 76 77 78
{
	struct socket *sock = sk->sk_socket;

	if (unlikely(sk->sk_prot == &tcp_prot)) {
		/* we are being invoked after mptcp_accept() has
		 * accepted a non-mp-capable flow: sk is a tcp_sk,
		 * not an mptcp one.
		 *
		 * Hand the socket over to tcp so all further socket ops
		 * bypass mptcp.
		 */
		sock->ops = &inet_stream_ops;
79
		return true;
F
Florian Westphal 已提交
80 81 82
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	} else if (unlikely(sk->sk_prot == &tcpv6_prot)) {
		sock->ops = &inet6_stream_ops;
83
		return true;
F
Florian Westphal 已提交
84 85 86
#endif
	}

87
	return false;
F
Florian Westphal 已提交
88 89
}

90
static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk)
91 92 93
{
	sock_owned_by_me((const struct sock *)msk);

94
	if (likely(!__mptcp_check_fallback(msk)))
95 96
		return NULL;

97
	return msk->first;
98 99
}

100
static int __mptcp_socket_create(struct mptcp_sock *msk)
101 102 103 104 105 106 107 108
{
	struct mptcp_subflow_context *subflow;
	struct sock *sk = (struct sock *)msk;
	struct socket *ssock;
	int err;

	err = mptcp_subflow_create_socket(sk, &ssock);
	if (err)
109
		return err;
110

111
	msk->first = ssock->sk;
112 113
	msk->subflow = ssock;
	subflow = mptcp_subflow_ctx(ssock->sk);
114
	list_add(&subflow->node, &msk->conn_list);
P
Paolo Abeni 已提交
115
	sock_hold(ssock->sk);
116 117
	subflow->request_mptcp = 1;

118 119 120 121 122
	/* accept() will wait on first subflow sk_wq, and we always wakes up
	 * via msk->sk_socket
	 */
	RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq);

123
	return 0;
124 125
}

126 127 128 129 130 131
static void mptcp_drop(struct sock *sk, struct sk_buff *skb)
{
	sk_drops_add(sk, skb);
	__kfree_skb(skb);
}

132 133 134 135 136 137 138 139 140 141
static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to,
			       struct sk_buff *from)
{
	bool fragstolen;
	int delta;

	if (MPTCP_SKB_CB(from)->offset ||
	    !skb_try_coalesce(to, from, &fragstolen, &delta))
		return false;

P
Paolo Abeni 已提交
142 143 144
	pr_debug("colesced seq %llx into %llx new len %d new end seq %llx",
		 MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq,
		 to->len, MPTCP_SKB_CB(from)->end_seq);
145
	MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq;
146 147 148 149 150 151
	kfree_skb_partial(from, fragstolen);
	atomic_add(delta, &sk->sk_rmem_alloc);
	sk_mem_charge(sk, delta);
	return true;
}

152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to,
				   struct sk_buff *from)
{
	if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq)
		return false;

	return mptcp_try_coalesce((struct sock *)msk, to, from);
}

/* "inspired" by tcp_data_queue_ofo(), main differences:
 * - use mptcp seqs
 * - don't cope with sacks
 */
static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb)
{
	struct sock *sk = (struct sock *)msk;
	struct rb_node **p, *parent;
	u64 seq, end_seq, max_seq;
	struct sk_buff *skb1;

	seq = MPTCP_SKB_CB(skb)->map_seq;
	end_seq = MPTCP_SKB_CB(skb)->end_seq;
174
	max_seq = READ_ONCE(msk->rcv_wnd_sent);
175

P
Paolo Abeni 已提交
176 177
	pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq,
		 RB_EMPTY_ROOT(&msk->out_of_order_queue));
178
	if (after64(end_seq, max_seq)) {
179 180
		/* out of window */
		mptcp_drop(sk, skb);
181 182 183
		pr_debug("oow by %lld, rcv_wnd_sent %llu\n",
			 (unsigned long long)end_seq - (unsigned long)max_seq,
			 (unsigned long long)msk->rcv_wnd_sent);
P
Paolo Abeni 已提交
184
		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW);
185 186 187 188
		return;
	}

	p = &msk->out_of_order_queue.rb_node;
P
Paolo Abeni 已提交
189
	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE);
190 191 192 193 194 195 196 197 198 199
	if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) {
		rb_link_node(&skb->rbnode, NULL, p);
		rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);
		msk->ooo_last_skb = skb;
		goto end;
	}

	/* with 2 subflows, adding at end of ooo queue is quite likely
	 * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
	 */
P
Paolo Abeni 已提交
200 201 202
	if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) {
		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
203
		return;
P
Paolo Abeni 已提交
204
	}
205 206 207

	/* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
	if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) {
P
Paolo Abeni 已提交
208
		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
		parent = &msk->ooo_last_skb->rbnode;
		p = &parent->rb_right;
		goto insert;
	}

	/* Find place to insert this segment. Handle overlaps on the way. */
	parent = NULL;
	while (*p) {
		parent = *p;
		skb1 = rb_to_skb(parent);
		if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
			p = &parent->rb_left;
			continue;
		}
		if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) {
			if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) {
				/* All the bits are present. Drop. */
				mptcp_drop(sk, skb);
P
Paolo Abeni 已提交
227
				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
				return;
			}
			if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
				/* partial overlap:
				 *     |     skb      |
				 *  |     skb1    |
				 * continue traversing
				 */
			} else {
				/* skb's seq == skb1's seq and skb covers skb1.
				 * Replace skb1 with skb.
				 */
				rb_replace_node(&skb1->rbnode, &skb->rbnode,
						&msk->out_of_order_queue);
				mptcp_drop(sk, skb1);
P
Paolo Abeni 已提交
243
				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
244 245 246
				goto merge_right;
			}
		} else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) {
P
Paolo Abeni 已提交
247
			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
248 249 250 251
			return;
		}
		p = &parent->rb_right;
	}
P
Paolo Abeni 已提交
252

253 254 255 256 257 258 259 260 261 262 263 264
insert:
	/* Insert segment into RB tree. */
	rb_link_node(&skb->rbnode, parent, p);
	rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);

merge_right:
	/* Remove other segments covered by skb. */
	while ((skb1 = skb_rb_next(skb)) != NULL) {
		if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq))
			break;
		rb_erase(&skb1->rbnode, &msk->out_of_order_queue);
		mptcp_drop(sk, skb1);
P
Paolo Abeni 已提交
265
		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
266 267 268 269 270 271 272 273 274 275 276 277 278
	}
	/* If there is no skb after us, we are the last_skb ! */
	if (!skb1)
		msk->ooo_last_skb = skb;

end:
	skb_condense(skb);
	skb_set_owner_r(skb, sk);
}

static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,
			     struct sk_buff *skb, unsigned int offset,
			     size_t copy_len)
279
{
280
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
281
	struct sock *sk = (struct sock *)msk;
282
	struct sk_buff *tail;
283 284 285

	__skb_unlink(skb, &ssk->sk_receive_queue);

286 287
	skb_ext_reset(skb);
	skb_orphan(skb);
288

289 290 291 292 293 294 295 296 297
	/* try to fetch required memory from subflow */
	if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
		if (ssk->sk_forward_alloc < skb->truesize)
			goto drop;
		__sk_mem_reclaim(ssk, skb->truesize);
		if (!sk_rmem_schedule(sk, skb, skb->truesize))
			goto drop;
	}

298 299 300 301 302 303
	/* the skb map_seq accounts for the skb offset:
	 * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq
	 * value
	 */
	MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow);
	MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len;
304
	MPTCP_SKB_CB(skb)->offset = offset;
305

306 307
	if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) {
		/* in sequence */
308
		WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len);
309 310 311
		tail = skb_peek_tail(&sk->sk_receive_queue);
		if (tail && mptcp_try_coalesce(sk, tail, skb))
			return true;
312

313 314 315 316 317 318 319 320 321 322 323
		skb_set_owner_r(skb, sk);
		__skb_queue_tail(&sk->sk_receive_queue, skb);
		return true;
	} else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) {
		mptcp_data_queue_ofo(msk, skb);
		return false;
	}

	/* old data, keep it simple and drop the whole pkt, sender
	 * will retransmit as needed, if needed.
	 */
P
Paolo Abeni 已提交
324
	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
325
drop:
326 327
	mptcp_drop(sk, skb);
	return false;
328 329
}

330 331 332 333 334 335 336 337
static void mptcp_stop_timer(struct sock *sk)
{
	struct inet_connection_sock *icsk = inet_csk(sk);

	sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
	mptcp_sk(sk)->timer_ival = 0;
}

P
Paolo Abeni 已提交
338 339 340 341 342 343 344 345 346 347 348 349 350
static void mptcp_close_wake_up(struct sock *sk)
{
	if (sock_flag(sk, SOCK_DEAD))
		return;

	sk->sk_state_change(sk);
	if (sk->sk_shutdown == SHUTDOWN_MASK ||
	    sk->sk_state == TCP_CLOSE)
		sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
	else
		sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
}

351
static bool mptcp_pending_data_fin_ack(struct sock *sk)
352 353 354
{
	struct mptcp_sock *msk = mptcp_sk(sk);

355 356 357 358 359 360 361 362 363
	return !__mptcp_check_fallback(msk) &&
	       ((1 << sk->sk_state) &
		(TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) &&
	       msk->write_seq == READ_ONCE(msk->snd_una);
}

static void mptcp_check_data_fin_ack(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
364 365

	/* Look for an acknowledged DATA_FIN */
366
	if (mptcp_pending_data_fin_ack(sk)) {
367 368 369 370 371 372 373 374 375 376 377 378 379 380
		mptcp_stop_timer(sk);

		WRITE_ONCE(msk->snd_data_fin_enable, 0);

		switch (sk->sk_state) {
		case TCP_FIN_WAIT1:
			inet_sk_state_store(sk, TCP_FIN_WAIT2);
			break;
		case TCP_CLOSING:
		case TCP_LAST_ACK:
			inet_sk_state_store(sk, TCP_CLOSE);
			break;
		}

P
Paolo Abeni 已提交
381
		mptcp_close_wake_up(sk);
382 383 384
	}
}

385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	if (READ_ONCE(msk->rcv_data_fin) &&
	    ((1 << sk->sk_state) &
	     (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) {
		u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq);

		if (msk->ack_seq == rcv_data_fin_seq) {
			if (seq)
				*seq = rcv_data_fin_seq;

			return true;
		}
	}

	return false;
}

static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk)
{
	long tout = ssk && inet_csk(ssk)->icsk_pending ?
				      inet_csk(ssk)->icsk_timeout - jiffies : 0;

	if (tout <= 0)
		tout = mptcp_sk(sk)->timer_ival;
	mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;
}

415 416 417 418 419 420 421 422 423 424 425 426
static bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)
{
	struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

	/* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */
	if (subflow->request_join && !subflow->fully_established)
		return false;

	/* only send if our side has not closed yet */
	return ((1 << ssk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT));
}

427 428 429 430 431 432 433
static bool tcp_can_send_ack(const struct sock *ssk)
{
	return !((1 << inet_sk_state_load(ssk)) &
	       (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE));
}

static void mptcp_send_ack(struct mptcp_sock *msk)
434 435 436 437 438 439
{
	struct mptcp_subflow_context *subflow;

	mptcp_for_each_subflow(msk, subflow) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

440 441
		lock_sock(ssk);
		if (tcp_can_send_ack(ssk))
442
			tcp_send_ack(ssk);
443
		release_sock(ssk);
444
	}
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
}

static bool mptcp_subflow_cleanup_rbuf(struct sock *ssk)
{
	int ret;

	lock_sock(ssk);
	ret = tcp_can_send_ack(ssk);
	if (ret)
		tcp_cleanup_rbuf(ssk, 1);
	release_sock(ssk);
	return ret;
}

static void mptcp_cleanup_rbuf(struct mptcp_sock *msk)
{
461
	struct sock *ack_hint = READ_ONCE(msk->ack_hint);
462 463 464
	struct mptcp_subflow_context *subflow;

	/* if the hinted ssk is still active, try to use it */
465
	if (likely(ack_hint)) {
466 467 468
		mptcp_for_each_subflow(msk, subflow) {
			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

469
			if (ack_hint == ssk && mptcp_subflow_cleanup_rbuf(ssk))
470 471
				return;
		}
472
	}
473 474 475 476 477

	/* otherwise pick the first active subflow */
	mptcp_for_each_subflow(msk, subflow)
		if (mptcp_subflow_cleanup_rbuf(mptcp_subflow_tcp_sock(subflow)))
			return;
478 479 480
}

static bool mptcp_check_data_fin(struct sock *sk)
481 482 483
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	u64 rcv_data_fin_seq;
484
	bool ret = false;
485 486

	if (__mptcp_check_fallback(msk) || !msk->first)
487
		return ret;
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502

	/* Need to ack a DATA_FIN received from a peer while this side
	 * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2.
	 * msk->rcv_data_fin was set when parsing the incoming options
	 * at the subflow level and the msk lock was not held, so this
	 * is the first opportunity to act on the DATA_FIN and change
	 * the msk state.
	 *
	 * If we are caught up to the sequence number of the incoming
	 * DATA_FIN, send the DATA_ACK now and do state transition.  If
	 * not caught up, do nothing and let the recv code send DATA_ACK
	 * when catching up.
	 */

	if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) {
503
		WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1);
504 505 506
		WRITE_ONCE(msk->rcv_data_fin, 0);

		sk->sk_shutdown |= RCV_SHUTDOWN;
507 508
		smp_mb__before_atomic(); /* SHUTDOWN must be visible first */
		set_bit(MPTCP_DATA_READY, &msk->flags);
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525

		switch (sk->sk_state) {
		case TCP_ESTABLISHED:
			inet_sk_state_store(sk, TCP_CLOSE_WAIT);
			break;
		case TCP_FIN_WAIT1:
			inet_sk_state_store(sk, TCP_CLOSING);
			break;
		case TCP_FIN_WAIT2:
			inet_sk_state_store(sk, TCP_CLOSE);
			break;
		default:
			/* Other states not expected */
			WARN_ON_ONCE(1);
			break;
		}

526
		ret = true;
527
		mptcp_set_timeout(sk, NULL);
528
		mptcp_send_ack(msk);
P
Paolo Abeni 已提交
529
		mptcp_close_wake_up(sk);
530
	}
531
	return ret;
532 533
}

534 535 536 537 538
static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
					   struct sock *ssk,
					   unsigned int *bytes)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
539
	struct sock *sk = (struct sock *)msk;
540 541 542 543
	unsigned int moved = 0;
	bool more_data_avail;
	struct tcp_sock *tp;
	bool done = false;
544 545 546 547 548 549 550 551 552 553 554 555
	int sk_rbuf;

	sk_rbuf = READ_ONCE(sk->sk_rcvbuf);

	if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
		int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf);

		if (unlikely(ssk_rbuf > sk_rbuf)) {
			WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf);
			sk_rbuf = ssk_rbuf;
		}
	}
556

557
	pr_debug("msk=%p ssk=%p", msk, ssk);
558 559 560 561 562 563 564 565 566 567 568 569
	tp = tcp_sk(ssk);
	do {
		u32 map_remaining, offset;
		u32 seq = tp->copied_seq;
		struct sk_buff *skb;
		bool fin;

		/* try to move as much data as available */
		map_remaining = subflow->map_data_len -
				mptcp_subflow_get_map_offset(subflow);

		skb = skb_peek(&ssk->sk_receive_queue);
570 571 572 573 574 575 576
		if (!skb) {
			/* if no data is found, a racing workqueue/recvmsg
			 * already processed the new data, stop here or we
			 * can enter an infinite loop
			 */
			if (!moved)
				done = true;
577
			break;
578
		}
579

580 581 582 583 584 585 586 587 588
		if (__mptcp_check_fallback(msk)) {
			/* if we are running under the workqueue, TCP could have
			 * collapsed skbs between dummy map creation and now
			 * be sure to adjust the size
			 */
			map_remaining = skb->len;
			subflow->map_data_len = skb->len;
		}

589 590 591 592 593 594 595 596 597 598 599 600 601
		offset = seq - TCP_SKB_CB(skb)->seq;
		fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
		if (fin) {
			done = true;
			seq++;
		}

		if (offset < skb->len) {
			size_t len = skb->len - offset;

			if (tp->urg_data)
				done = true;

602 603
			if (__mptcp_move_skb(msk, ssk, skb, offset, len))
				moved += len;
604 605 606 607 608 609 610 611 612 613 614 615
			seq += len;

			if (WARN_ON_ONCE(map_remaining < len))
				break;
		} else {
			WARN_ON_ONCE(!fin);
			sk_eat_skb(ssk, skb);
			done = true;
		}

		WRITE_ONCE(tp->copied_seq, seq);
		more_data_avail = mptcp_subflow_data_available(ssk);
616

617
		if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) {
618 619 620
			done = true;
			break;
		}
621
	} while (more_data_avail);
622
	WRITE_ONCE(msk->ack_hint, ssk);
623

624
	*bytes += moved;
625 626 627
	return done;
}

628
static bool __mptcp_ofo_queue(struct mptcp_sock *msk)
629 630 631 632 633 634 635 636
{
	struct sock *sk = (struct sock *)msk;
	struct sk_buff *skb, *tail;
	bool moved = false;
	struct rb_node *p;
	u64 end_seq;

	p = rb_first(&msk->out_of_order_queue);
P
Paolo Abeni 已提交
637
	pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue));
638 639 640 641 642 643 644 645 646 647 648
	while (p) {
		skb = rb_to_skb(p);
		if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq))
			break;

		p = rb_next(p);
		rb_erase(&skb->rbnode, &msk->out_of_order_queue);

		if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq,
				      msk->ack_seq))) {
			mptcp_drop(sk, skb);
P
Paolo Abeni 已提交
649
			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
650 651 652 653 654 655 656 657 658
			continue;
		}

		end_seq = MPTCP_SKB_CB(skb)->end_seq;
		tail = skb_peek_tail(&sk->sk_receive_queue);
		if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) {
			int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq;

			/* skip overlapping data, if any */
P
Paolo Abeni 已提交
659 660 661
			pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d",
				 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq,
				 delta);
662 663 664 665 666 667 668 669 670
			MPTCP_SKB_CB(skb)->offset += delta;
			__skb_queue_tail(&sk->sk_receive_queue, skb);
		}
		msk->ack_seq = end_seq;
		moved = true;
	}
	return moved;
}

671 672 673
/* In most cases we will be able to lock the mptcp socket.  If its already
 * owned, we need to defer to the work queue to avoid ABBA deadlock.
 */
674
static void move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)
675 676 677 678
{
	struct sock *sk = (struct sock *)msk;
	unsigned int moved = 0;

679 680
	if (inet_sk_state_load(sk) == TCP_CLOSE)
		return;
681

682
	mptcp_data_lock(sk);
683

684 685
	__mptcp_move_skbs_from_subflow(msk, ssk, &moved);
	__mptcp_ofo_queue(msk);
686

687 688 689 690 691 692 693 694
	/* If the moves have caught up with the DATA_FIN sequence number
	 * it's time to ack the DATA_FIN and change socket state, but
	 * this is not a good place to change state. Let the workqueue
	 * do it.
	 */
	if (mptcp_pending_data_fin(sk, NULL))
		mptcp_schedule_work(sk);
	mptcp_data_unlock(sk);
695 696 697
}

void mptcp_data_ready(struct sock *sk, struct sock *ssk)
698
{
699
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
700
	struct mptcp_sock *msk = mptcp_sk(sk);
701
	int sk_rbuf, ssk_rbuf;
702
	bool wake;
703

704 705 706 707 708 709 710
	/* move_skbs_to_msk below can legitly clear the data_avail flag,
	 * but we will need later to properly woke the reader, cache its
	 * value
	 */
	wake = subflow->data_avail == MPTCP_SUBFLOW_DATA_AVAIL;
	if (wake)
		set_bit(MPTCP_DATA_READY, &msk->flags);
711

712 713 714 715 716 717 718
	ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf);
	sk_rbuf = READ_ONCE(sk->sk_rcvbuf);
	if (unlikely(ssk_rbuf > sk_rbuf))
		sk_rbuf = ssk_rbuf;

	/* over limit? can't append more skbs to msk */
	if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf)
719 720
		goto wake;

721
	move_skbs_to_msk(msk, ssk);
722 723

wake:
724 725
	if (wake)
		sk->sk_data_ready(sk);
726 727
}

728
void __mptcp_flush_join_list(struct mptcp_sock *msk)
729 730 731 732 733 734 735 736 737
{
	if (likely(list_empty(&msk->join_list)))
		return;

	spin_lock_bh(&msk->join_list_lock);
	list_splice_tail_init(&msk->join_list, &msk->conn_list);
	spin_unlock_bh(&msk->join_list_lock);
}

738 739 740 741 742 743 744 745 746 747
static bool mptcp_timer_pending(struct sock *sk)
{
	return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);
}

static void mptcp_reset_timer(struct sock *sk)
{
	struct inet_connection_sock *icsk = inet_csk(sk);
	unsigned long tout;

P
Paolo Abeni 已提交
748 749 750 751
	/* prevent rescheduling on close */
	if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE))
		return;

752 753 754 755 756 757 758
	/* should never be called with mptcp level timer cleared */
	tout = READ_ONCE(mptcp_sk(sk)->timer_ival);
	if (WARN_ON_ONCE(!tout))
		tout = TCP_RTO_MIN;
	sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);
}

P
Paolo Abeni 已提交
759 760 761 762 763 764 765 766 767 768 769 770 771
bool mptcp_schedule_work(struct sock *sk)
{
	if (inet_sk_state_load(sk) != TCP_CLOSE &&
	    schedule_work(&mptcp_sk(sk)->work)) {
		/* each subflow already holds a reference to the sk, and the
		 * workqueue is invoked by a subflow, so sk can't go away here.
		 */
		sock_hold(sk);
		return true;
	}
	return false;
}

772 773
void mptcp_subflow_eof(struct sock *sk)
{
P
Paolo Abeni 已提交
774 775
	if (!test_and_set_bit(MPTCP_WORK_EOF, &mptcp_sk(sk)->flags))
		mptcp_schedule_work(sk);
776 777
}

778 779 780 781 782 783 784 785
static void mptcp_check_for_eof(struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow;
	struct sock *sk = (struct sock *)msk;
	int receivers = 0;

	mptcp_for_each_subflow(msk, subflow)
		receivers += !subflow->rx_eof;
P
Paolo Abeni 已提交
786 787
	if (receivers)
		return;
788

P
Paolo Abeni 已提交
789
	if (!(sk->sk_shutdown & RCV_SHUTDOWN)) {
790 791 792 793 794 795 796 797 798
		/* hopefully temporary hack: propagate shutdown status
		 * to msk, when all subflows agree on it
		 */
		sk->sk_shutdown |= RCV_SHUTDOWN;

		smp_mb__before_atomic(); /* SHUTDOWN must be visible first */
		set_bit(MPTCP_DATA_READY, &msk->flags);
		sk->sk_data_ready(sk);
	}
P
Paolo Abeni 已提交
799 800 801 802 803 804

	switch (sk->sk_state) {
	case TCP_ESTABLISHED:
		inet_sk_state_store(sk, TCP_CLOSE_WAIT);
		break;
	case TCP_FIN_WAIT1:
805 806 807
		inet_sk_state_store(sk, TCP_CLOSING);
		break;
	case TCP_FIN_WAIT2:
P
Paolo Abeni 已提交
808 809 810 811 812 813
		inet_sk_state_store(sk, TCP_CLOSE);
		break;
	default:
		return;
	}
	mptcp_close_wake_up(sk);
814 815
}

816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow;
	struct sock *sk = (struct sock *)msk;

	sock_owned_by_me(sk);

	mptcp_for_each_subflow(msk, subflow) {
		if (subflow->data_avail)
			return mptcp_subflow_tcp_sock(subflow);
	}

	return NULL;
}

831 832 833
static bool mptcp_skb_can_collapse_to(u64 write_seq,
				      const struct sk_buff *skb,
				      const struct mptcp_ext *mpext)
834 835 836 837
{
	if (!tcp_skb_can_collapse_to(skb))
		return false;

838 839 840 841 842
	/* can collapse only if MPTCP level sequence is in order and this
	 * mapping has not been xmitted yet
	 */
	return mpext && mpext->data_seq + mpext->data_len == write_seq &&
	       !mpext->frozen;
843 844
}

845 846 847 848 849
static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk,
				       const struct page_frag *pfrag,
				       const struct mptcp_data_frag *df)
{
	return df && pfrag->page == df->page &&
850
		pfrag->size - pfrag->offset > 0 &&
851 852 853
		df->data_seq + df->data_len == msk->write_seq;
}

854
static int mptcp_wmem_with_overhead(struct sock *sk, int size)
P
Paolo Abeni 已提交
855
{
856 857 858 859 860 861 862 863 864
	struct mptcp_sock *msk = mptcp_sk(sk);
	int ret, skbs;

	ret = size + ((sizeof(struct mptcp_data_frag) * size) >> PAGE_SHIFT);
	skbs = (msk->tx_pending_data + size) / msk->size_goal_cache;
	if (skbs < msk->skb_tx_cache.qlen)
		return ret;

	return ret + (skbs - msk->skb_tx_cache.qlen) * SKB_TRUESIZE(MAX_TCP_HEADER);
P
Paolo Abeni 已提交
865 866 867 868
}

static void __mptcp_wmem_reserve(struct sock *sk, int size)
{
869
	int amount = mptcp_wmem_with_overhead(sk, size);
P
Paolo Abeni 已提交
870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925
	struct mptcp_sock *msk = mptcp_sk(sk);

	WARN_ON_ONCE(msk->wmem_reserved);
	if (amount <= sk->sk_forward_alloc)
		goto reserve;

	/* under memory pressure try to reserve at most a single page
	 * otherwise try to reserve the full estimate and fallback
	 * to a single page before entering the error path
	 */
	if ((tcp_under_memory_pressure(sk) && amount > PAGE_SIZE) ||
	    !sk_wmem_schedule(sk, amount)) {
		if (amount <= PAGE_SIZE)
			goto nomem;

		amount = PAGE_SIZE;
		if (!sk_wmem_schedule(sk, amount))
			goto nomem;
	}

reserve:
	msk->wmem_reserved = amount;
	sk->sk_forward_alloc -= amount;
	return;

nomem:
	/* we will wait for memory on next allocation */
	msk->wmem_reserved = -1;
}

static void __mptcp_update_wmem(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	if (!msk->wmem_reserved)
		return;

	if (msk->wmem_reserved < 0)
		msk->wmem_reserved = 0;
	if (msk->wmem_reserved > 0) {
		sk->sk_forward_alloc += msk->wmem_reserved;
		msk->wmem_reserved = 0;
	}
}

static bool mptcp_wmem_alloc(struct sock *sk, int size)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	/* check for pre-existing error condition */
	if (msk->wmem_reserved < 0)
		return false;

	if (msk->wmem_reserved >= size)
		goto account;

926 927 928
	mptcp_data_lock(sk);
	if (!sk_wmem_schedule(sk, size)) {
		mptcp_data_unlock(sk);
P
Paolo Abeni 已提交
929
		return false;
930
	}
P
Paolo Abeni 已提交
931 932 933

	sk->sk_forward_alloc -= size;
	msk->wmem_reserved += size;
934
	mptcp_data_unlock(sk);
P
Paolo Abeni 已提交
935 936 937 938 939 940

account:
	msk->wmem_reserved -= size;
	return true;
}

941 942 943 944 945 946 947 948 949
static void mptcp_wmem_uncharge(struct sock *sk, int size)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	if (msk->wmem_reserved < 0)
		msk->wmem_reserved = 0;
	msk->wmem_reserved += size;
}

950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968
static void mptcp_mem_reclaim_partial(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	/* if we are experiencing a transint allocation error,
	 * the forward allocation memory has been already
	 * released
	 */
	if (msk->wmem_reserved < 0)
		return;

	mptcp_data_lock(sk);
	sk->sk_forward_alloc += msk->wmem_reserved;
	sk_mem_reclaim_partial(sk);
	msk->wmem_reserved = sk->sk_forward_alloc;
	sk->sk_forward_alloc = 0;
	mptcp_data_unlock(sk);
}

969 970 971
static void dfrag_uncharge(struct sock *sk, int len)
{
	sk_mem_uncharge(sk, len);
972
	sk_wmem_queued_add(sk, -len);
973 974 975
}

static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)
976
{
977 978
	int len = dfrag->data_len + dfrag->overhead;

979
	list_del(&dfrag->list);
980
	dfrag_uncharge(sk, len);
981 982 983
	put_page(dfrag->page);
}

984
static void __mptcp_clean_una(struct sock *sk)
985 986 987
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct mptcp_data_frag *dtmp, *dfrag;
988
	bool cleaned = false;
989 990 991 992 993 994
	u64 snd_una;

	/* on fallback we just need to ignore snd_una, as this is really
	 * plain TCP
	 */
	if (__mptcp_check_fallback(msk))
995
		msk->snd_una = READ_ONCE(msk->snd_nxt);
996

997
	snd_una = msk->snd_una;
998 999 1000 1001
	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {
		if (after64(dfrag->data_seq + dfrag->data_len, snd_una))
			break;

1002 1003
		if (WARN_ON_ONCE(dfrag == msk->first_pending))
			break;
1004 1005 1006 1007
		dfrag_clear(sk, dfrag);
		cleaned = true;
	}

1008 1009
	dfrag = mptcp_rtx_head(sk);
	if (dfrag && after64(snd_una, dfrag->data_seq)) {
1010 1011
		u64 delta = snd_una - dfrag->data_seq;

1012
		if (WARN_ON_ONCE(delta > dfrag->already_sent))
1013
			goto out;
1014 1015

		dfrag->data_seq += delta;
1016
		dfrag->offset += delta;
1017
		dfrag->data_len -= delta;
1018
		dfrag->already_sent -= delta;
1019 1020 1021 1022 1023

		dfrag_uncharge(sk, delta);
		cleaned = true;
	}

1024
out:
1025 1026 1027 1028 1029
	if (cleaned) {
		if (tcp_under_memory_pressure(sk)) {
			__mptcp_update_wmem(sk);
			sk_mem_reclaim_partial(sk);
		}
1030

1031 1032 1033 1034 1035 1036 1037
		if (sk_stream_is_writeable(sk)) {
			/* pairs with memory barrier in mptcp_poll */
			smp_mb();
			if (test_and_clear_bit(MPTCP_NOSPACE, &msk->flags))
				sk_stream_write_space(sk);
		}
	}
1038

1039 1040 1041 1042 1043
	if (snd_una == READ_ONCE(msk->snd_nxt)) {
		if (msk->timer_ival)
			mptcp_stop_timer(sk);
	} else {
		mptcp_reset_timer(sk);
1044 1045 1046
	}
}

1047
static void mptcp_enter_memory_pressure(struct sock *sk)
1048
{
1049 1050 1051 1052
	struct mptcp_subflow_context *subflow;
	struct mptcp_sock *msk = mptcp_sk(sk);
	bool first = true;

1053
	sk_stream_moderate_sndbuf(sk);
1054 1055 1056 1057 1058 1059 1060 1061
	mptcp_for_each_subflow(msk, subflow) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

		if (first)
			tcp_enter_memory_pressure(ssk);
		sk_stream_moderate_sndbuf(ssk);
		first = false;
	}
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
}

/* ensure we get enough memory for the frag hdr, beyond some minimal amount of
 * data
 */
static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
{
	if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag),
					pfrag, sk->sk_allocation)))
		return true;

	mptcp_enter_memory_pressure(sk);
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
	return false;
}

static struct mptcp_data_frag *
mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag,
		      int orig_offset)
{
	int offset = ALIGN(orig_offset, sizeof(long));
	struct mptcp_data_frag *dfrag;

	dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);
	dfrag->data_len = 0;
	dfrag->data_seq = msk->write_seq;
	dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);
	dfrag->offset = offset + sizeof(struct mptcp_data_frag);
1089
	dfrag->already_sent = 0;
1090 1091 1092 1093 1094
	dfrag->page = pfrag->page;

	return dfrag;
}

1095 1096 1097
struct mptcp_sendmsg_info {
	int mss_now;
	int size_goal;
1098 1099 1100
	u16 limit;
	u16 sent;
	unsigned int flags;
1101 1102
};

1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
static int mptcp_check_allowed_size(struct mptcp_sock *msk, u64 data_seq,
				    int avail_size)
{
	u64 window_end = mptcp_wnd_end(msk);

	if (__mptcp_check_fallback(msk))
		return avail_size;

	if (!before64(data_seq + avail_size, window_end)) {
		u64 allowed_size = window_end - data_seq;

		return min_t(unsigned int, allowed_size, avail_size);
	}

	return avail_size;
}

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp)
{
	struct skb_ext *mpext = __skb_ext_alloc(gfp);

	if (!mpext)
		return false;
	__skb_ext_set(skb, SKB_EXT_MPTCP, mpext);
	return true;
}

1130
static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp)
1131 1132 1133
{
	struct sk_buff *skb;

1134
	skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
1135
	if (likely(skb)) {
1136
		if (likely(__mptcp_add_ext(skb, gfp))) {
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
			skb_reserve(skb, MAX_TCP_HEADER);
			skb->reserved_tailroom = skb->end - skb->tail;
			return skb;
		}
		__kfree_skb(skb);
	} else {
		mptcp_enter_memory_pressure(sk);
	}
	return NULL;
}

static bool mptcp_tx_cache_refill(struct sock *sk, int size,
				  struct sk_buff_head *skbs, int *total_ts)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct sk_buff *skb;
	int space_needed;

	if (unlikely(tcp_under_memory_pressure(sk))) {
		mptcp_mem_reclaim_partial(sk);

		/* under pressure pre-allocate at most a single skb */
		if (msk->skb_tx_cache.qlen)
			return true;
		space_needed = msk->size_goal_cache;
	} else {
		space_needed = msk->tx_pending_data + size -
			       msk->skb_tx_cache.qlen * msk->size_goal_cache;
	}

	while (space_needed > 0) {
1168
		skb = __mptcp_do_alloc_tx_skb(sk, sk->sk_allocation);
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186
		if (unlikely(!skb)) {
			/* under memory pressure, try to pass the caller a
			 * single skb to allow forward progress
			 */
			while (skbs->qlen > 1) {
				skb = __skb_dequeue_tail(skbs);
				__kfree_skb(skb);
			}
			return skbs->qlen > 0;
		}

		*total_ts += skb->truesize;
		__skb_queue_tail(skbs, skb);
		space_needed -= msk->size_goal_cache;
	}
	return true;
}

1187
static bool __mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp)
1188 1189 1190 1191 1192 1193 1194
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct sk_buff *skb;

	if (ssk->sk_tx_skb_cache) {
		skb = ssk->sk_tx_skb_cache;
		if (unlikely(!skb_ext_find(skb, SKB_EXT_MPTCP) &&
1195
			     !__mptcp_add_ext(skb, gfp)))
1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215
			return false;
		return true;
	}

	skb = skb_peek(&msk->skb_tx_cache);
	if (skb) {
		if (likely(sk_wmem_schedule(ssk, skb->truesize))) {
			skb = __skb_dequeue(&msk->skb_tx_cache);
			if (WARN_ON_ONCE(!skb))
				return false;

			mptcp_wmem_uncharge(sk, skb->truesize);
			ssk->sk_tx_skb_cache = skb;
			return true;
		}

		/* over memory limit, no point to try to allocate a new skb */
		return false;
	}

1216
	skb = __mptcp_do_alloc_tx_skb(sk, gfp);
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
	if (!skb)
		return false;

	if (likely(sk_wmem_schedule(ssk, skb->truesize))) {
		ssk->sk_tx_skb_cache = skb;
		return true;
	}
	kfree_skb(skb);
	return false;
}

static bool mptcp_must_reclaim_memory(struct sock *sk, struct sock *ssk)
{
	return !ssk->sk_tx_skb_cache &&
	       !skb_peek(&mptcp_sk(sk)->skb_tx_cache) &&
	       tcp_under_memory_pressure(sk);
}

static bool mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk)
{
	if (unlikely(mptcp_must_reclaim_memory(sk, ssk)))
		mptcp_mem_reclaim_partial(sk);
1239
	return __mptcp_alloc_tx_skb(sk, ssk, sk->sk_allocation);
1240 1241
}

1242
static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
1243
			      struct mptcp_data_frag *dfrag,
1244
			      struct mptcp_sendmsg_info *info)
1245
{
1246
	u64 data_seq = dfrag->data_seq + info->sent;
1247
	struct mptcp_sock *msk = mptcp_sk(sk);
1248
	bool zero_window_probe = false;
1249
	struct mptcp_ext *mpext = NULL;
1250
	struct sk_buff *skb, *tail;
1251 1252
	bool can_collapse = false;
	int avail_size;
1253
	size_t ret = 0;
1254

1255 1256 1257 1258 1259
	pr_debug("msk=%p ssk=%p sending dfrag at seq=%lld len=%d already sent=%d",
		 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent);

	/* compute send limit */
	info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags);
1260
	avail_size = info->size_goal;
1261
	msk->size_goal_cache = info->size_goal;
1262 1263 1264 1265 1266 1267 1268 1269
	skb = tcp_write_queue_tail(ssk);
	if (skb) {
		/* Limit the write to the size available in the
		 * current skb, if any, so that we create at most a new skb.
		 * Explicitly tells TCP internals to avoid collapsing on later
		 * queue management operation, to avoid breaking the ext <->
		 * SSN association set here
		 */
1270
		mpext = skb_ext_find(skb, SKB_EXT_MPTCP);
1271
		can_collapse = (info->size_goal - skb->len > 0) &&
1272
			 mptcp_skb_can_collapse_to(data_seq, skb, mpext);
1273 1274 1275
		if (!can_collapse)
			TCP_SKB_CB(skb)->eor = 1;
		else
1276
			avail_size = info->size_goal - skb->len;
1277
	}
1278

1279 1280 1281
	/* Zero window and all data acked? Probe. */
	avail_size = mptcp_check_allowed_size(msk, data_seq, avail_size);
	if (avail_size == 0) {
1282 1283 1284
		u64 snd_una = READ_ONCE(msk->snd_una);

		if (skb || snd_una != msk->snd_nxt)
1285 1286
			return 0;
		zero_window_probe = true;
1287
		data_seq = snd_una - 1;
1288 1289 1290
		avail_size = 1;
	}

1291 1292 1293
	if (WARN_ON_ONCE(info->sent > info->limit ||
			 info->limit > dfrag->data_len))
		return 0;
1294

1295 1296 1297
	ret = info->limit - info->sent;
	tail = tcp_build_frag(ssk, avail_size, info->flags, dfrag->page,
			      dfrag->offset + info->sent, &ret);
P
Paolo Abeni 已提交
1298 1299 1300
	if (!tail) {
		tcp_remove_empty_skb(sk, tcp_write_queue_tail(ssk));
		return -ENOMEM;
1301
	}
1302

P
Paolo Abeni 已提交
1303
	/* if the tail skb is still the cached one, collapsing really happened.
1304
	 */
P
Paolo Abeni 已提交
1305
	if (skb == tail) {
1306 1307
		WARN_ON_ONCE(!can_collapse);
		mpext->data_len += ret;
1308
		WARN_ON_ONCE(zero_window_probe);
1309 1310 1311
		goto out;
	}

1312 1313 1314 1315 1316
	mpext = skb_ext_find(tail, SKB_EXT_MPTCP);
	if (WARN_ON_ONCE(!mpext)) {
		/* should never reach here, stream corrupted */
		return -EINVAL;
	}
1317 1318

	memset(mpext, 0, sizeof(*mpext));
1319
	mpext->data_seq = data_seq;
1320 1321 1322 1323 1324 1325 1326 1327 1328
	mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;
	mpext->data_len = ret;
	mpext->use_map = 1;
	mpext->dsn64 = 1;

	pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d",
		 mpext->data_seq, mpext->subflow_seq, mpext->data_len,
		 mpext->dsn64);

1329 1330 1331 1332 1333 1334
	if (zero_window_probe) {
		mptcp_subflow_ctx(ssk)->rel_write_seq += ret;
		mpext->frozen = 1;
		ret = 0;
		tcp_push_pending_frames(ssk);
	}
1335
out:
1336 1337 1338 1339
	mptcp_subflow_ctx(ssk)->rel_write_seq += ret;
	return ret;
}

1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
#define MPTCP_SEND_BURST_SIZE		((1 << 16) - \
					 sizeof(struct tcphdr) - \
					 MAX_TCP_OPTION_SPACE - \
					 sizeof(struct ipv6hdr) - \
					 sizeof(struct frag_hdr))

struct subflow_send_info {
	struct sock *ssk;
	u64 ratio;
};

P
Paolo Abeni 已提交
1351 1352
static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk,
					   u32 *sndbuf)
1353
{
1354
	struct subflow_send_info send_info[2];
1355
	struct mptcp_subflow_context *subflow;
1356 1357 1358 1359
	int i, nr_active = 0;
	struct sock *ssk;
	u64 ratio;
	u32 pace;
1360

1361
	sock_owned_by_me((struct sock *)msk);
1362

P
Paolo Abeni 已提交
1363
	*sndbuf = 0;
1364 1365
	if (__mptcp_check_fallback(msk)) {
		if (!msk->first)
1366
			return NULL;
1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
		*sndbuf = msk->first->sk_sndbuf;
		return sk_stream_memory_free(msk->first) ? msk->first : NULL;
	}

	/* re-use last subflow, if the burst allow that */
	if (msk->last_snd && msk->snd_burst > 0 &&
	    sk_stream_memory_free(msk->last_snd) &&
	    mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) {
		mptcp_for_each_subflow(msk, subflow) {
			ssk =  mptcp_subflow_tcp_sock(subflow);
			*sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf);
1378
		}
1379 1380
		return msk->last_snd;
	}
1381

1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
	/* pick the subflow with the lower wmem/wspace ratio */
	for (i = 0; i < 2; ++i) {
		send_info[i].ssk = NULL;
		send_info[i].ratio = -1;
	}
	mptcp_for_each_subflow(msk, subflow) {
		ssk =  mptcp_subflow_tcp_sock(subflow);
		if (!mptcp_subflow_active(subflow))
			continue;

		nr_active += !subflow->backup;
P
Paolo Abeni 已提交
1393
		*sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf);
1394 1395
		if (!sk_stream_memory_free(subflow->tcp_sock))
			continue;
1396

1397 1398
		pace = READ_ONCE(ssk->sk_pacing_rate);
		if (!pace)
1399 1400
			continue;

1401 1402 1403 1404 1405 1406
		ratio = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32,
				pace);
		if (ratio < send_info[subflow->backup].ratio) {
			send_info[subflow->backup].ssk = ssk;
			send_info[subflow->backup].ratio = ratio;
		}
1407 1408
	}

1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
	pr_debug("msk=%p nr_active=%d ssk=%p:%lld backup=%p:%lld",
		 msk, nr_active, send_info[0].ssk, send_info[0].ratio,
		 send_info[1].ssk, send_info[1].ratio);

	/* pick the best backup if no other subflow is active */
	if (!nr_active)
		send_info[0].ssk = send_info[1].ssk;

	if (send_info[0].ssk) {
		msk->last_snd = send_info[0].ssk;
		msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE,
				       sk_stream_wspace(msk->last_snd));
		return msk->last_snd;
	}
	return NULL;
1424 1425
}

1426 1427 1428 1429 1430 1431 1432 1433 1434
static void mptcp_push_release(struct sock *sk, struct sock *ssk,
			       struct mptcp_sendmsg_info *info)
{
	mptcp_set_timeout(sk, ssk);
	tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal);
	release_sock(ssk);
}

static void mptcp_push_pending(struct sock *sk, unsigned int flags)
M
Mat Martineau 已提交
1435
{
1436
	struct sock *prev_ssk = NULL, *ssk = NULL;
M
Mat Martineau 已提交
1437
	struct mptcp_sock *msk = mptcp_sk(sk);
1438
	struct mptcp_sendmsg_info info = {
1439
				.flags = flags,
1440
	};
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
	struct mptcp_data_frag *dfrag;
	int len, copied = 0;
	u32 sndbuf;

	while ((dfrag = mptcp_send_head(sk))) {
		info.sent = dfrag->already_sent;
		info.limit = dfrag->data_len;
		len = dfrag->data_len - dfrag->already_sent;
		while (len > 0) {
			int ret = 0;

			prev_ssk = ssk;
			__mptcp_flush_join_list(msk);
			ssk = mptcp_subflow_get_send(msk, &sndbuf);

			/* do auto tuning */
			if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK) &&
			    sndbuf > READ_ONCE(sk->sk_sndbuf))
				WRITE_ONCE(sk->sk_sndbuf, sndbuf);

			/* try to keep the subflow socket lock across
			 * consecutive xmit on the same socket
			 */
			if (ssk != prev_ssk && prev_ssk)
				mptcp_push_release(sk, prev_ssk, &info);
			if (!ssk)
				goto out;

			if (ssk != prev_ssk || !prev_ssk)
				lock_sock(ssk);

1472 1473 1474 1475 1476 1477 1478 1479 1480
			/* keep it simple and always provide a new skb for the
			 * subflow, even if we will not use it when collapsing
			 * on the pending one
			 */
			if (!mptcp_alloc_tx_skb(sk, ssk)) {
				mptcp_push_release(sk, ssk, &info);
				goto out;
			}

1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
			ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
			if (ret <= 0) {
				mptcp_push_release(sk, ssk, &info);
				goto out;
			}

			info.sent += ret;
			dfrag->already_sent += ret;
			msk->snd_nxt += ret;
			msk->snd_burst -= ret;
1491
			msk->tx_pending_data -= ret;
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
			copied += ret;
			len -= ret;
		}
		WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
	}

	/* at this point we held the socket lock for the last subflow we used */
	if (ssk)
		mptcp_push_release(sk, ssk, &info);

out:
1503 1504 1505 1506
	if (copied) {
		/* start the timer, if it's not pending */
		if (!mptcp_timer_pending(sk))
			mptcp_reset_timer(sk);
1507
		__mptcp_check_send_data_fin(sk);
1508
	}
1509 1510
}

1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct mptcp_sendmsg_info info;
	struct mptcp_data_frag *dfrag;
	int len, copied = 0;

	info.flags = 0;
	while ((dfrag = mptcp_send_head(sk))) {
		info.sent = dfrag->already_sent;
		info.limit = dfrag->data_len;
		len = dfrag->data_len - dfrag->already_sent;
		while (len > 0) {
			int ret = 0;

			/* do auto tuning */
			if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK) &&
			    ssk->sk_sndbuf > READ_ONCE(sk->sk_sndbuf))
				WRITE_ONCE(sk->sk_sndbuf, ssk->sk_sndbuf);

			if (unlikely(mptcp_must_reclaim_memory(sk, ssk))) {
				__mptcp_update_wmem(sk);
				sk_mem_reclaim_partial(sk);
			}
			if (!__mptcp_alloc_tx_skb(sk, ssk, GFP_ATOMIC))
				goto out;

			ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
			if (ret <= 0)
				goto out;

			info.sent += ret;
			dfrag->already_sent += ret;
			msk->snd_nxt += ret;
			msk->snd_burst -= ret;
			msk->tx_pending_data -= ret;
			copied += ret;
			len -= ret;
		}
		WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
	}

out:
	/* __mptcp_alloc_tx_skb could have released some wmem and we are
	 * not going to flush it via release_sock()
	 */
	__mptcp_update_wmem(sk);
	if (copied) {
		mptcp_set_timeout(sk, ssk);
		tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,
			 info.size_goal);
		if (msk->snd_data_fin_enable &&
		    msk->snd_nxt + 1 == msk->write_seq)
			mptcp_schedule_work(sk);
	}
}

1568 1569 1570
static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
1571
	struct page_frag *pfrag;
1572
	size_t copied = 0;
1573
	int ret = 0;
1574
	long timeo;
M
Mat Martineau 已提交
1575 1576 1577 1578

	if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
		return -EOPNOTSUPP;

P
Paolo Abeni 已提交
1579
	mptcp_lock_sock(sk, __mptcp_wmem_reserve(sk, len));
1580 1581 1582 1583 1584 1585 1586 1587 1588

	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

	if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
		ret = sk_stream_wait_connect(sk, &timeo);
		if (ret)
			goto out;
	}

1589
	pfrag = sk_page_frag(sk);
1590

1591
	while (msg_data_left(msg)) {
1592
		int total_ts, frag_truesize = 0;
1593
		struct mptcp_data_frag *dfrag;
1594
		struct sk_buff_head skbs;
1595 1596
		bool dfrag_collapsed;
		size_t psize, offset;
1597

1598 1599
		if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) {
			ret = -EPIPE;
1600 1601
			goto out;
		}
P
Paolo Abeni 已提交
1602

1603 1604 1605 1606 1607 1608
		/* reuse tail pfrag, if possible, or carve a new one from the
		 * page allocator
		 */
		dfrag = mptcp_pending_tail(sk);
		dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);
		if (!dfrag_collapsed) {
1609 1610 1611
			if (!sk_stream_memory_free(sk))
				goto wait_for_memory;

1612 1613 1614 1615 1616
			if (!mptcp_page_frag_refill(sk, pfrag))
				goto wait_for_memory;

			dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset);
			frag_truesize = dfrag->overhead;
1617
		}
1618

1619 1620 1621
		/* we do not bound vs wspace, to allow a single packet.
		 * memory accounting will prevent execessive memory usage
		 * anyway
1622
		 */
1623 1624 1625
		offset = dfrag->offset + dfrag->data_len;
		psize = pfrag->size - offset;
		psize = min_t(size_t, psize, msg_data_left(msg));
1626 1627 1628
		total_ts = psize + frag_truesize;
		__skb_queue_head_init(&skbs);
		if (!mptcp_tx_cache_refill(sk, psize, &skbs, &total_ts))
1629 1630
			goto wait_for_memory;

1631 1632 1633 1634 1635 1636
		if (!mptcp_wmem_alloc(sk, total_ts)) {
			__skb_queue_purge(&skbs);
			goto wait_for_memory;
		}

		skb_queue_splice_tail(&skbs, &msk->skb_tx_cache);
1637 1638
		if (copy_page_from_iter(dfrag->page, offset, psize,
					&msg->msg_iter) != psize) {
1639
			mptcp_wmem_uncharge(sk, psize + frag_truesize);
1640 1641
			ret = -EFAULT;
			goto out;
1642 1643
		}

1644 1645 1646 1647 1648 1649 1650 1651 1652
		/* data successfully copied into the write queue */
		copied += psize;
		dfrag->data_len += psize;
		frag_truesize += psize;
		pfrag->offset += frag_truesize;
		WRITE_ONCE(msk->write_seq, msk->write_seq + psize);

		/* charge data on mptcp pending queue to the msk socket
		 * Note: we charge such data both to sk and ssk
1653
		 */
1654 1655 1656 1657 1658 1659
		sk_wmem_queued_add(sk, frag_truesize);
		if (!dfrag_collapsed) {
			get_page(dfrag->page);
			list_add_tail(&dfrag->list, &msk->rtx_queue);
			if (!msk->first_pending)
				WRITE_ONCE(msk->first_pending, dfrag);
1660
		}
1661 1662 1663
		pr_debug("msk=%p dfrag at seq=%lld len=%d sent=%d new=%d", msk,
			 dfrag->data_seq, dfrag->data_len, dfrag->already_sent,
			 !dfrag_collapsed);
1664

1665
		continue;
1666

1667
wait_for_memory:
1668 1669
		set_bit(MPTCP_NOSPACE, &msk->flags);
		mptcp_push_pending(sk, msg->msg_flags);
1670 1671 1672
		ret = sk_stream_wait_memory(sk, &timeo);
		if (ret)
			goto out;
1673
	}
1674

1675 1676
	if (copied) {
		msk->tx_pending_data += copied;
1677
		mptcp_push_pending(sk, msg->msg_flags);
1678
	}
1679

1680
out:
1681
	release_sock(sk);
1682
	return copied ? : ret;
M
Mat Martineau 已提交
1683 1684
}

1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
static void mptcp_wait_data(struct sock *sk, long *timeo)
{
	DEFINE_WAIT_FUNC(wait, woken_wake_function);
	struct mptcp_sock *msk = mptcp_sk(sk);

	add_wait_queue(sk_sleep(sk), &wait);
	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);

	sk_wait_event(sk, timeo,
		      test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait);

	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
	remove_wait_queue(sk_sleep(sk), &wait);
}

1700 1701 1702 1703 1704 1705 1706
static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
				struct msghdr *msg,
				size_t len)
{
	struct sk_buff *skb;
	int copied = 0;

1707
	while ((skb = skb_peek(&msk->receive_queue)) != NULL) {
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726
		u32 offset = MPTCP_SKB_CB(skb)->offset;
		u32 data_len = skb->len - offset;
		u32 count = min_t(size_t, len - copied, data_len);
		int err;

		err = skb_copy_datagram_msg(skb, offset, msg, count);
		if (unlikely(err < 0)) {
			if (!copied)
				return err;
			break;
		}

		copied += count;

		if (count < data_len) {
			MPTCP_SKB_CB(skb)->offset += count;
			break;
		}

1727 1728 1729 1730
		/* we will bulk release the skb memory later */
		skb->destructor = NULL;
		msk->rmem_released += skb->truesize;
		__skb_unlink(skb, &msk->receive_queue);
1731 1732 1733 1734 1735 1736 1737 1738 1739
		__kfree_skb(skb);

		if (copied >= len)
			break;
	}

	return copied;
}

1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
/* receive buffer autotuning.  See tcp_rcv_space_adjust for more information.
 *
 * Only difference: Use highest rtt estimate of the subflows in use.
 */
static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied)
{
	struct mptcp_subflow_context *subflow;
	struct sock *sk = (struct sock *)msk;
	u32 time, advmss = 1;
	u64 rtt_us, mstamp;

	sock_owned_by_me(sk);

	if (copied <= 0)
		return;

	msk->rcvq_space.copied += copied;

	mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC);
	time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time);

	rtt_us = msk->rcvq_space.rtt_us;
	if (rtt_us && time < (rtt_us >> 3))
		return;

	rtt_us = 0;
	mptcp_for_each_subflow(msk, subflow) {
		const struct tcp_sock *tp;
		u64 sf_rtt_us;
		u32 sf_advmss;

		tp = tcp_sk(mptcp_subflow_tcp_sock(subflow));

		sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us);
		sf_advmss = READ_ONCE(tp->advmss);

		rtt_us = max(sf_rtt_us, rtt_us);
		advmss = max(sf_advmss, advmss);
	}

	msk->rcvq_space.rtt_us = rtt_us;
	if (time < (rtt_us >> 3) || rtt_us == 0)
		return;

	if (msk->rcvq_space.copied <= msk->rcvq_space.space)
		goto new_measure;

	if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf &&
	    !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
		int rcvmem, rcvbuf;
		u64 rcvwin, grow;

		rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss;

		grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space);

		do_div(grow, msk->rcvq_space.space);
		rcvwin += (grow << 1);

		rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER);
		while (tcp_win_from_space(sk, rcvmem) < advmss)
			rcvmem += 128;

		do_div(rcvwin, advmss);
		rcvbuf = min_t(u64, rcvwin * rcvmem,
			       sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);

		if (rcvbuf > sk->sk_rcvbuf) {
			u32 window_clamp;

			window_clamp = tcp_win_from_space(sk, rcvbuf);
			WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);

			/* Make subflows follow along.  If we do not do this, we
			 * get drops at subflow level if skbs can't be moved to
			 * the mptcp rx queue fast enough (announced rcv_win can
			 * exceed ssk->sk_rcvbuf).
			 */
			mptcp_for_each_subflow(msk, subflow) {
				struct sock *ssk;
1820
				bool slow;
1821 1822

				ssk = mptcp_subflow_tcp_sock(subflow);
1823
				slow = lock_sock_fast(ssk);
1824 1825
				WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf);
				tcp_sk(ssk)->window_clamp = window_clamp;
1826 1827
				tcp_cleanup_rbuf(ssk, 1);
				unlock_sock_fast(ssk, slow);
1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
			}
		}
	}

	msk->rcvq_space.space = msk->rcvq_space.copied;
new_measure:
	msk->rcvq_space.copied = 0;
	msk->rcvq_space.time = mstamp;
}

1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856
static void __mptcp_update_rmem(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	if (!msk->rmem_released)
		return;

	atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc);
	sk_mem_uncharge(sk, msk->rmem_released);
	msk->rmem_released = 0;
}

static void __mptcp_splice_receive_queue(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue);
}

1857
static bool __mptcp_move_skbs(struct mptcp_sock *msk, unsigned int rcv)
1858
{
1859
	struct sock *sk = (struct sock *)msk;
1860
	unsigned int moved = 0;
1861
	bool ret, done;
1862 1863

	__mptcp_flush_join_list(msk);
1864 1865
	do {
		struct sock *ssk = mptcp_subflow_recv_lookup(msk);
1866
		bool slowpath;
1867

1868 1869 1870 1871 1872
		/* we can have data pending in the subflows only if the msk
		 * receive buffer was full at subflow_data_ready() time,
		 * that is an unlikely slow path.
		 */
		if (likely(!ssk))
1873 1874
			break;

1875
		slowpath = lock_sock_fast(ssk);
1876
		mptcp_data_lock(sk);
1877
		done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
1878
		mptcp_data_unlock(sk);
1879 1880 1881 1882 1883
		if (moved && rcv) {
			WRITE_ONCE(msk->rmem_pending, min(rcv, moved));
			tcp_cleanup_rbuf(ssk, 1);
			WRITE_ONCE(msk->rmem_pending, 0);
		}
1884
		unlock_sock_fast(ssk, slowpath);
1885 1886
	} while (!done);

1887 1888 1889 1890 1891 1892 1893 1894 1895
	/* acquire the data lock only if some input data is pending */
	ret = moved > 0;
	if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) ||
	    !skb_queue_empty_lockless(&sk->sk_receive_queue)) {
		mptcp_data_lock(sk);
		__mptcp_update_rmem(sk);
		ret |= __mptcp_ofo_queue(msk);
		__mptcp_splice_receive_queue(sk);
		mptcp_data_unlock(sk);
1896
	}
1897 1898 1899
	if (ret)
		mptcp_check_data_fin((struct sock *)msk);
	return !skb_queue_empty(&msk->receive_queue);
1900 1901
}

M
Mat Martineau 已提交
1902 1903 1904 1905
static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
			 int nonblock, int flags, int *addr_len)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
1906
	int copied = 0;
1907 1908
	int target;
	long timeo;
M
Mat Martineau 已提交
1909 1910 1911 1912

	if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT))
		return -EOPNOTSUPP;

1913
	mptcp_lock_sock(sk, __mptcp_splice_receive_queue(sk));
1914 1915 1916 1917 1918
	if (unlikely(sk->sk_state == TCP_LISTEN)) {
		copied = -ENOTCONN;
		goto out_err;
	}

1919 1920 1921 1922 1923
	timeo = sock_rcvtimeo(sk, nonblock);

	len = min_t(size_t, len, INT_MAX);
	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);

1924
	while (copied < len) {
1925
		int bytes_read, old_space;
1926

1927 1928 1929 1930 1931 1932
		bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied);
		if (unlikely(bytes_read < 0)) {
			if (!copied)
				copied = bytes_read;
			goto out_err;
		}
1933

1934
		copied += bytes_read;
1935

1936
		if (skb_queue_empty(&msk->receive_queue) &&
1937
		    __mptcp_move_skbs(msk, len - copied))
1938
			continue;
1939

1940 1941 1942
		/* be sure to advertise window change */
		old_space = READ_ONCE(msk->old_wspace);
		if ((tcp_space(sk) - old_space) >= old_space)
1943
			mptcp_cleanup_rbuf(msk);
1944

1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963
		/* only the master socket status is relevant here. The exit
		 * conditions mirror closely tcp_recvmsg()
		 */
		if (copied >= target)
			break;

		if (copied) {
			if (sk->sk_err ||
			    sk->sk_state == TCP_CLOSE ||
			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
			    !timeo ||
			    signal_pending(current))
				break;
		} else {
			if (sk->sk_err) {
				copied = sock_error(sk);
				break;
			}

1964 1965 1966
			if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags))
				mptcp_check_for_eof(msk);

1967 1968 1969 1970 1971 1972
			if (sk->sk_shutdown & RCV_SHUTDOWN) {
				/* race breaker: the shutdown could be after the
				 * previous receive queue check
				 */
				if (__mptcp_move_skbs(msk, len - copied))
					continue;
1973
				break;
1974
			}
1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993

			if (sk->sk_state == TCP_CLOSE) {
				copied = -ENOTCONN;
				break;
			}

			if (!timeo) {
				copied = -EAGAIN;
				break;
			}

			if (signal_pending(current)) {
				copied = sock_intr_errno(timeo);
				break;
			}
		}

		pr_debug("block timeout %ld", timeo);
		mptcp_wait_data(sk, &timeo);
1994 1995
	}

1996 1997
	if (skb_queue_empty_lockless(&sk->sk_receive_queue) &&
	    skb_queue_empty(&msk->receive_queue)) {
1998
		/* entire backlog drained, clear DATA_READY. */
1999
		clear_bit(MPTCP_DATA_READY, &msk->flags);
2000

2001 2002
		/* .. race-breaker: ssk might have gotten new data
		 * after last __mptcp_move_skbs() returned false.
2003
		 */
2004
		if (unlikely(__mptcp_move_skbs(msk, 0)))
2005
			set_bit(MPTCP_DATA_READY, &msk->flags);
2006 2007 2008
	} else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) {
		/* data to read but mptcp_wait_data() cleared DATA_READY */
		set_bit(MPTCP_DATA_READY, &msk->flags);
2009
	}
2010
out_err:
2011 2012
	pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d",
		 msk, test_bit(MPTCP_DATA_READY, &msk->flags),
2013
		 skb_queue_empty_lockless(&sk->sk_receive_queue), copied);
2014 2015
	mptcp_rcv_space_adjust(msk, copied);

2016
	release_sock(sk);
2017 2018 2019
	return copied;
}

2020 2021 2022 2023
static void mptcp_retransmit_handler(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

2024 2025
	set_bit(MPTCP_WORK_RTX, &msk->flags);
	mptcp_schedule_work(sk);
2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
}

static void mptcp_retransmit_timer(struct timer_list *t)
{
	struct inet_connection_sock *icsk = from_timer(icsk, t,
						       icsk_retransmit_timer);
	struct sock *sk = &icsk->icsk_inet.sk;

	bh_lock_sock(sk);
	if (!sock_owned_by_user(sk)) {
		mptcp_retransmit_handler(sk);
	} else {
		/* delegate our work to tcp_release_cb() */
		if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED,
				      &sk->sk_tsq_flags))
			sock_hold(sk);
	}
	bh_unlock_sock(sk);
	sock_put(sk);
}

P
Paolo Abeni 已提交
2047 2048 2049 2050 2051
static void mptcp_timeout_timer(struct timer_list *t)
{
	struct sock *sk = from_timer(sk, t, sk_timer);

	mptcp_schedule_work(sk);
2052
	sock_put(sk);
P
Paolo Abeni 已提交
2053 2054
}

2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
/* Find an idle subflow.  Return NULL if there is unacked data at tcp
 * level.
 *
 * A backup subflow is returned only if that is the only kind available.
 */
static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow;
	struct sock *backup = NULL;

	sock_owned_by_me((const struct sock *)msk);

2067
	if (__mptcp_check_fallback(msk))
2068
		return NULL;
2069

2070 2071 2072
	mptcp_for_each_subflow(msk, subflow) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

2073 2074 2075
		if (!mptcp_subflow_active(subflow))
			continue;

2076
		/* still data outstanding at TCP level?  Don't retransmit. */
2077 2078 2079
		if (!tcp_write_queue_empty(ssk)) {
			if (inet_csk(ssk)->icsk_ca_state >= TCP_CA_Loss)
				continue;
2080
			return NULL;
2081
		}
2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094

		if (subflow->backup) {
			if (!backup)
				backup = ssk;
			continue;
		}

		return ssk;
	}

	return backup;
}

2095 2096 2097 2098 2099 2100 2101 2102
/* subflow sockets can be either outgoing (connect) or incoming
 * (accept).
 *
 * Outgoing subflows use in-kernel sockets.
 * Incoming subflows do not have their own 'struct socket' allocated,
 * so we need to use tcp_close() after detaching them from the mptcp
 * parent socket.
 */
2103
void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
P
Paolo Abeni 已提交
2104
		       struct mptcp_subflow_context *subflow)
2105
{
P
Paolo Abeni 已提交
2106 2107
	bool dispose_socket = false;
	struct socket *sock;
2108 2109 2110

	list_del(&subflow->node);

P
Paolo Abeni 已提交
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127
	lock_sock(ssk);

	/* if we are invoked by the msk cleanup code, the subflow is
	 * already orphaned
	 */
	sock = ssk->sk_socket;
	if (sock) {
		dispose_socket = sock != sk->sk_socket;
		sock_orphan(ssk);
	}

	/* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops
	 * the ssk has been already destroyed, we just need to release the
	 * reference owned by msk;
	 */
	if (!inet_csk(ssk)->icsk_ulp_ops) {
		kfree_rcu(subflow, rcu);
2128
	} else {
P
Paolo Abeni 已提交
2129 2130 2131 2132 2133 2134
		/* otherwise ask tcp do dispose of ssk and subflow ctx */
		subflow->disposable = 1;
		__tcp_close(ssk, 0);

		/* close acquired an extra ref */
		__sock_put(ssk);
2135
	}
P
Paolo Abeni 已提交
2136 2137 2138 2139 2140
	release_sock(ssk);
	if (dispose_socket)
		iput(SOCK_INODE(sock));

	sock_put(ssk);
M
Mat Martineau 已提交
2141 2142
}

P
Paolo Abeni 已提交
2143 2144 2145 2146 2147
static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu)
{
	return 0;
}

2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
static void pm_work(struct mptcp_sock *msk)
{
	struct mptcp_pm_data *pm = &msk->pm;

	spin_lock_bh(&msk->pm.lock);

	pr_debug("msk=%p status=%x", msk, pm->status);
	if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) {
		pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED);
		mptcp_pm_nl_add_addr_received(msk);
	}
2159 2160 2161 2162
	if (pm->status & BIT(MPTCP_PM_ADD_ADDR_SEND_ACK)) {
		pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_SEND_ACK);
		mptcp_pm_nl_add_addr_send_ack(msk);
	}
2163 2164 2165 2166
	if (pm->status & BIT(MPTCP_PM_RM_ADDR_RECEIVED)) {
		pm->status &= ~BIT(MPTCP_PM_RM_ADDR_RECEIVED);
		mptcp_pm_nl_rm_addr_received(msk);
	}
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178
	if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) {
		pm->status &= ~BIT(MPTCP_PM_ESTABLISHED);
		mptcp_pm_nl_fully_established(msk);
	}
	if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) {
		pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED);
		mptcp_pm_nl_subflow_established(msk);
	}

	spin_unlock_bh(&msk->pm.lock);
}

P
Paolo Abeni 已提交
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
static void __mptcp_close_subflow(struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow, *tmp;

	list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

		if (inet_sk_state_load(ssk) != TCP_CLOSE)
			continue;

P
Paolo Abeni 已提交
2189
		__mptcp_close_ssk((struct sock *)msk, ssk, subflow);
P
Paolo Abeni 已提交
2190 2191 2192
	}
}

P
Paolo Abeni 已提交
2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
static bool mptcp_check_close_timeout(const struct sock *sk)
{
	s32 delta = tcp_jiffies32 - inet_csk(sk)->icsk_mtup.probe_timestamp;
	struct mptcp_subflow_context *subflow;

	if (delta >= TCP_TIMEWAIT_LEN)
		return true;

	/* if all subflows are in closed status don't bother with additional
	 * timeout
	 */
	mptcp_for_each_subflow(mptcp_sk(sk), subflow) {
		if (inet_sk_state_load(mptcp_subflow_tcp_sock(subflow)) !=
		    TCP_CLOSE)
			return false;
	}
	return true;
}

P
Paolo Abeni 已提交
2212 2213 2214
static void mptcp_worker(struct work_struct *work)
{
	struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work);
2215
	struct sock *ssk, *sk = &msk->sk.icsk_inet.sk;
2216
	struct mptcp_sendmsg_info info = {};
2217 2218
	struct mptcp_data_frag *dfrag;
	size_t copied = 0;
P
Paolo Abeni 已提交
2219
	int state, ret;
P
Paolo Abeni 已提交
2220 2221

	lock_sock(sk);
P
Paolo Abeni 已提交
2222 2223 2224 2225
	state = sk->sk_state;
	if (unlikely(state == TCP_CLOSE))
		goto unlock;

2226
	mptcp_check_data_fin_ack(sk);
2227
	__mptcp_flush_join_list(msk);
P
Paolo Abeni 已提交
2228 2229 2230
	if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
		__mptcp_close_subflow(msk);

2231 2232 2233
	if (msk->pm.status)
		pm_work(msk);

2234 2235 2236
	if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags))
		mptcp_check_for_eof(msk);

2237
	__mptcp_check_send_data_fin(sk);
2238 2239
	mptcp_check_data_fin(sk);

P
Paolo Abeni 已提交
2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251
	/* if the msk data is completely acked, or the socket timedout,
	 * there is no point in keeping around an orphaned sk
	 */
	if (sock_flag(sk, SOCK_DEAD) &&
	    (mptcp_check_close_timeout(sk) ||
	    (state != sk->sk_state &&
	    ((1 << inet_sk_state_load(sk)) & (TCPF_CLOSE | TCPF_FIN_WAIT2))))) {
		inet_sk_state_store(sk, TCP_CLOSE);
		__mptcp_destroy_sock(sk);
		goto unlock;
	}

2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264
	if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags))
		goto unlock;

	dfrag = mptcp_rtx_head(sk);
	if (!dfrag)
		goto unlock;

	ssk = mptcp_subflow_get_retrans(msk);
	if (!ssk)
		goto reset_unlock;

	lock_sock(ssk);

2265 2266 2267 2268
	/* limit retransmission to the bytes already sent on some subflows */
	info.sent = 0;
	info.limit = dfrag->already_sent;
	while (info.sent < dfrag->already_sent) {
2269 2270 2271
		if (!mptcp_alloc_tx_skb(sk, ssk))
			break;

2272
		ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
2273
		if (ret <= 0)
2274 2275
			break;

2276
		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS);
2277
		copied += ret;
2278
		info.sent += ret;
2279 2280
	}
	if (copied)
2281 2282
		tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,
			 info.size_goal);
2283 2284 2285 2286 2287 2288 2289 2290 2291

	mptcp_set_timeout(sk, ssk);
	release_sock(ssk);

reset_unlock:
	if (!mptcp_timer_pending(sk))
		mptcp_reset_timer(sk);

unlock:
P
Paolo Abeni 已提交
2292 2293 2294 2295
	release_sock(sk);
	sock_put(sk);
}

2296
static int __mptcp_init_sock(struct sock *sk)
M
Mat Martineau 已提交
2297
{
2298 2299
	struct mptcp_sock *msk = mptcp_sk(sk);

2300 2301
	spin_lock_init(&msk->join_list_lock);

2302
	INIT_LIST_HEAD(&msk->conn_list);
2303
	INIT_LIST_HEAD(&msk->join_list);
2304
	INIT_LIST_HEAD(&msk->rtx_queue);
P
Paolo Abeni 已提交
2305
	INIT_WORK(&msk->work, mptcp_worker);
2306
	__skb_queue_head_init(&msk->receive_queue);
2307
	__skb_queue_head_init(&msk->skb_tx_cache);
2308
	msk->out_of_order_queue = RB_ROOT;
2309
	msk->first_pending = NULL;
P
Paolo Abeni 已提交
2310
	msk->wmem_reserved = 0;
2311
	msk->rmem_released = 0;
2312 2313
	msk->tx_pending_data = 0;
	msk->size_goal_cache = TCP_BASE_MSS;
2314

2315
	msk->ack_hint = NULL;
2316
	msk->first = NULL;
P
Paolo Abeni 已提交
2317
	inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss;
2318

2319 2320
	mptcp_pm_data_init(msk);

2321 2322
	/* re-use the csk retrans timer for MPTCP-level retrans */
	timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0);
P
Paolo Abeni 已提交
2323
	timer_setup(&sk->sk_timer, mptcp_timeout_timer, 0);
M
Mat Martineau 已提交
2324 2325 2326
	return 0;
}

2327 2328
static int mptcp_init_sock(struct sock *sk)
{
2329 2330
	struct net *net = sock_net(sk);
	int ret;
2331

2332 2333 2334 2335
	ret = __mptcp_init_sock(sk);
	if (ret)
		return ret;

2336 2337 2338 2339 2340 2341
	if (!mptcp_is_enabled(net))
		return -ENOPROTOOPT;

	if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net))
		return -ENOMEM;

2342 2343 2344 2345
	ret = __mptcp_socket_create(mptcp_sk(sk));
	if (ret)
		return ret;

2346
	sk_sockets_allocated_inc(sk);
2347
	sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1];
P
Paolo Abeni 已提交
2348
	sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1];
2349

2350 2351 2352 2353 2354 2355 2356
	return 0;
}

static void __mptcp_clear_xmit(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct mptcp_data_frag *dtmp, *dfrag;
2357
	struct sk_buff *skb;
2358

2359
	WRITE_ONCE(msk->first_pending, NULL);
2360
	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)
2361
		dfrag_clear(sk, dfrag);
2362 2363 2364 2365
	while ((skb = __skb_dequeue(&msk->skb_tx_cache)) != NULL) {
		sk->sk_forward_alloc += skb->truesize;
		kfree_skb(skb);
	}
2366 2367
}

P
Paolo Abeni 已提交
2368 2369 2370 2371
static void mptcp_cancel_work(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

2372
	if (cancel_work_sync(&msk->work))
P
Paolo Abeni 已提交
2373
		__sock_put(sk);
P
Paolo Abeni 已提交
2374 2375
}

2376
void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how)
2377 2378 2379 2380 2381 2382 2383
{
	lock_sock(ssk);

	switch (ssk->sk_state) {
	case TCP_LISTEN:
		if (!(how & RCV_SHUTDOWN))
			break;
2384
		fallthrough;
2385 2386 2387 2388
	case TCP_SYN_SENT:
		tcp_disconnect(ssk, O_NONBLOCK);
		break;
	default:
2389 2390 2391 2392 2393 2394 2395 2396 2397
		if (__mptcp_check_fallback(mptcp_sk(sk))) {
			pr_debug("Fallback");
			ssk->sk_shutdown |= how;
			tcp_shutdown(ssk, how);
		} else {
			pr_debug("Sending DATA_FIN on subflow %p", ssk);
			mptcp_set_timeout(sk, ssk);
			tcp_send_ack(ssk);
		}
2398 2399 2400 2401 2402 2403
		break;
	}

	release_sock(ssk);
}

2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430
static const unsigned char new_state[16] = {
	/* current state:     new state:      action:	*/
	[0 /* (Invalid) */] = TCP_CLOSE,
	[TCP_ESTABLISHED]   = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
	[TCP_SYN_SENT]      = TCP_CLOSE,
	[TCP_SYN_RECV]      = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
	[TCP_FIN_WAIT1]     = TCP_FIN_WAIT1,
	[TCP_FIN_WAIT2]     = TCP_FIN_WAIT2,
	[TCP_TIME_WAIT]     = TCP_CLOSE,	/* should not happen ! */
	[TCP_CLOSE]         = TCP_CLOSE,
	[TCP_CLOSE_WAIT]    = TCP_LAST_ACK  | TCP_ACTION_FIN,
	[TCP_LAST_ACK]      = TCP_LAST_ACK,
	[TCP_LISTEN]        = TCP_CLOSE,
	[TCP_CLOSING]       = TCP_CLOSING,
	[TCP_NEW_SYN_RECV]  = TCP_CLOSE,	/* should not happen ! */
};

static int mptcp_close_state(struct sock *sk)
{
	int next = (int)new_state[sk->sk_state];
	int ns = next & TCP_STATE_MASK;

	inet_sk_state_store(sk, ns);

	return next & TCP_ACTION_FIN;
}

P
Paolo Abeni 已提交
2431
static void __mptcp_check_send_data_fin(struct sock *sk)
M
Mat Martineau 已提交
2432
{
P
Paolo Abeni 已提交
2433
	struct mptcp_subflow_context *subflow;
M
Mat Martineau 已提交
2434 2435
	struct mptcp_sock *msk = mptcp_sk(sk);

P
Paolo Abeni 已提交
2436 2437 2438
	pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu",
		 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk),
		 msk->snd_nxt, msk->write_seq);
2439

P
Paolo Abeni 已提交
2440 2441 2442 2443 2444 2445 2446 2447 2448
	/* we still need to enqueue subflows or not really shutting down,
	 * skip this
	 */
	if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq ||
	    mptcp_send_head(sk))
		return;

	WRITE_ONCE(msk->snd_nxt, msk->write_seq);

2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
	/* fallback socket will not get data_fin/ack, can move to the next
	 * state now
	 */
	if (__mptcp_check_fallback(msk)) {
		if ((1 << sk->sk_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) {
			inet_sk_state_store(sk, TCP_CLOSE);
			mptcp_close_wake_up(sk);
		} else if (sk->sk_state == TCP_FIN_WAIT1) {
			inet_sk_state_store(sk, TCP_FIN_WAIT2);
		}
2459 2460
	}

P
Paolo Abeni 已提交
2461 2462 2463
	__mptcp_flush_join_list(msk);
	mptcp_for_each_subflow(msk, subflow) {
		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
2464

P
Paolo Abeni 已提交
2465
		mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN);
2466
	}
P
Paolo Abeni 已提交
2467
}
2468

P
Paolo Abeni 已提交
2469 2470 2471
static void __mptcp_wr_shutdown(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
2472

P
Paolo Abeni 已提交
2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490
	pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d",
		 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state,
		 !!mptcp_send_head(sk));

	/* will be ignored by fallback sockets */
	WRITE_ONCE(msk->write_seq, msk->write_seq + 1);
	WRITE_ONCE(msk->snd_data_fin_enable, 1);

	__mptcp_check_send_data_fin(sk);
}

static void __mptcp_destroy_sock(struct sock *sk)
{
	struct mptcp_subflow_context *subflow, *tmp;
	struct mptcp_sock *msk = mptcp_sk(sk);
	LIST_HEAD(conn_list);

	pr_debug("msk=%p", msk);
M
Mat Martineau 已提交
2491

2492 2493 2494 2495 2496 2497
	/* be sure to always acquire the join list lock, to sync vs
	 * mptcp_finish_join().
	 */
	spin_lock_bh(&msk->join_list_lock);
	list_splice_tail_init(&msk->join_list, &msk->conn_list);
	spin_unlock_bh(&msk->join_list_lock);
2498 2499
	list_splice_init(&msk->conn_list, &conn_list);

2500
	sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer);
P
Paolo Abeni 已提交
2501 2502
	sk_stop_timer(sk, &sk->sk_timer);
	msk->pm.status = 0;
2503 2504

	list_for_each_entry_safe(subflow, tmp, &conn_list, node) {
2505
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
P
Paolo Abeni 已提交
2506
		__mptcp_close_ssk(sk, ssk, subflow);
M
Mat Martineau 已提交
2507 2508
	}

P
Paolo Abeni 已提交
2509
	sk->sk_prot->destroy(sk);
P
Paolo Abeni 已提交
2510

P
Paolo Abeni 已提交
2511
	WARN_ON_ONCE(msk->wmem_reserved);
2512
	WARN_ON_ONCE(msk->rmem_released);
P
Paolo Abeni 已提交
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
	sk_stream_kill_queues(sk);
	xfrm_sk_free_policy(sk);
	sk_refcnt_debug_release(sk);
	sock_put(sk);
}

static void mptcp_close(struct sock *sk, long timeout)
{
	struct mptcp_subflow_context *subflow;
	bool do_cancel_work = false;

	lock_sock(sk);
	sk->sk_shutdown = SHUTDOWN_MASK;

	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) {
		inet_sk_state_store(sk, TCP_CLOSE);
		goto cleanup;
	}
2531

P
Paolo Abeni 已提交
2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570
	if (mptcp_close_state(sk))
		__mptcp_wr_shutdown(sk);

	sk_stream_wait_close(sk, timeout);

cleanup:
	/* orphan all the subflows */
	inet_csk(sk)->icsk_mtup.probe_timestamp = tcp_jiffies32;
	list_for_each_entry(subflow, &mptcp_sk(sk)->conn_list, node) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
		bool slow, dispose_socket;
		struct socket *sock;

		slow = lock_sock_fast(ssk);
		sock = ssk->sk_socket;
		dispose_socket = sock && sock != sk->sk_socket;
		sock_orphan(ssk);
		unlock_sock_fast(ssk, slow);

		/* for the outgoing subflows we additionally need to free
		 * the associated socket
		 */
		if (dispose_socket)
			iput(SOCK_INODE(sock));
	}
	sock_orphan(sk);

	sock_hold(sk);
	pr_debug("msk=%p state=%d", sk, sk->sk_state);
	if (sk->sk_state == TCP_CLOSE) {
		__mptcp_destroy_sock(sk);
		do_cancel_work = true;
	} else {
		sk_reset_timer(sk, &sk->sk_timer, jiffies + TCP_TIMEWAIT_LEN);
	}
	release_sock(sk);
	if (do_cancel_work)
		mptcp_cancel_work(sk);
	sock_put(sk);
M
Mat Martineau 已提交
2571 2572
}

2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595
static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)
{
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
	struct ipv6_pinfo *msk6 = inet6_sk(msk);

	msk->sk_v6_daddr = ssk->sk_v6_daddr;
	msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;

	if (msk6 && ssk6) {
		msk6->saddr = ssk6->saddr;
		msk6->flow_label = ssk6->flow_label;
	}
#endif

	inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
	inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
	inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
	inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
	inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
	inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
}

2596 2597
static int mptcp_disconnect(struct sock *sk, int flags)
{
2598 2599 2600 2601 2602 2603
	/* Should never be called.
	 * inet_stream_connect() calls ->disconnect, but that
	 * refers to the subflow socket, not the mptcp one.
	 */
	WARN_ON_ONCE(1);
	return 0;
2604 2605
}

2606 2607 2608 2609 2610 2611 2612 2613 2614
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk)
{
	unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo);

	return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
}
#endif

2615
struct sock *mptcp_sk_clone(const struct sock *sk,
2616
			    const struct mptcp_options_received *mp_opt,
2617
			    struct request_sock *req)
2618
{
P
Paolo Abeni 已提交
2619
	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
2620
	struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC);
P
Paolo Abeni 已提交
2621 2622
	struct mptcp_sock *msk;
	u64 ack_seq;
2623 2624 2625 2626 2627 2628 2629 2630 2631

	if (!nsk)
		return NULL;

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	if (nsk->sk_family == AF_INET6)
		inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk);
#endif

P
Paolo Abeni 已提交
2632 2633 2634 2635 2636 2637
	__mptcp_init_sock(nsk);

	msk = mptcp_sk(nsk);
	msk->local_key = subflow_req->local_key;
	msk->token = subflow_req->token;
	msk->subflow = NULL;
2638
	WRITE_ONCE(msk->fully_established, false);
P
Paolo Abeni 已提交
2639 2640

	msk->write_seq = subflow_req->idsn + 1;
P
Paolo Abeni 已提交
2641
	msk->snd_nxt = msk->write_seq;
2642 2643
	msk->snd_una = msk->write_seq;
	msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd;
2644

2645
	if (mp_opt->mp_capable) {
P
Paolo Abeni 已提交
2646
		msk->can_ack = true;
2647
		msk->remote_key = mp_opt->sndr_key;
P
Paolo Abeni 已提交
2648 2649
		mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
		ack_seq++;
2650
		WRITE_ONCE(msk->ack_seq, ack_seq);
2651
		WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
P
Paolo Abeni 已提交
2652
	}
2653

2654
	sock_reset_flag(nsk, SOCK_RCU_FREE);
2655 2656
	/* will be fully established after successful MPC subflow creation */
	inet_sk_state_store(nsk, TCP_SYN_RECV);
P
Paolo Abeni 已提交
2657 2658 2659 2660
	bh_unlock_sock(nsk);

	/* keep a single reference */
	__sock_put(nsk);
2661 2662 2663
	return nsk;
}

2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677
void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk)
{
	const struct tcp_sock *tp = tcp_sk(ssk);

	msk->rcvq_space.copied = 0;
	msk->rcvq_space.rtt_us = 0;

	msk->rcvq_space.time = tp->tcp_mstamp;

	/* initial rcv_space offering made to peer */
	msk->rcvq_space.space = min_t(u32, tp->rcv_wnd,
				      TCP_INIT_CWND * tp->advmss);
	if (msk->rcvq_space.space == 0)
		msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT;
2678

2679
	WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd);
2680 2681
}

2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
				 bool kern)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *listener;
	struct sock *newsk;

	listener = __mptcp_nmpc_socket(msk);
	if (WARN_ON_ONCE(!listener)) {
		*err = -EINVAL;
		return NULL;
	}

	pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk));
	newsk = inet_csk_accept(listener->sk, flags, err, kern);
	if (!newsk)
		return NULL;

	pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk));
	if (sk_is_mptcp(newsk)) {
		struct mptcp_subflow_context *subflow;
		struct sock *new_mptcp_sock;

		subflow = mptcp_subflow_ctx(newsk);
P
Paolo Abeni 已提交
2706
		new_mptcp_sock = subflow->conn;
2707

P
Paolo Abeni 已提交
2708 2709 2710 2711 2712 2713
		/* is_mptcp should be false if subflow->conn is missing, see
		 * subflow_syn_recv_sock()
		 */
		if (WARN_ON_ONCE(!new_mptcp_sock)) {
			tcp_sk(newsk)->is_mptcp = 0;
			return newsk;
2714 2715
		}

P
Paolo Abeni 已提交
2716 2717
		/* acquire the 2nd reference for the owning socket */
		sock_hold(new_mptcp_sock);
2718
		newsk = new_mptcp_sock;
2719
		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK);
2720 2721 2722
	} else {
		MPTCP_INC_STATS(sock_net(sk),
				MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK);
2723 2724 2725 2726 2727
	}

	return newsk;
}

2728 2729
void mptcp_destroy_common(struct mptcp_sock *msk)
{
2730 2731
	struct sock *sk = (struct sock *)msk;

2732 2733
	__mptcp_clear_xmit(sk);

2734 2735 2736
	/* move to sk_receive_queue, sk_stream_kill_queues will purge it */
	skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue);

2737 2738 2739 2740 2741
	skb_rbtree_purge(&msk->out_of_order_queue);
	mptcp_token_destroy(msk);
	mptcp_pm_free_anno_list(msk);
}

2742 2743
static void mptcp_destroy(struct sock *sk)
{
2744 2745
	struct mptcp_sock *msk = mptcp_sk(sk);

2746
	mptcp_destroy_common(msk);
2747
	sk_sockets_allocated_dec(sk);
2748 2749
}

2750
static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname,
2751
				       sockptr_t optval, unsigned int optlen)
2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766
{
	struct sock *sk = (struct sock *)msk;
	struct socket *ssock;
	int ret;

	switch (optname) {
	case SO_REUSEPORT:
	case SO_REUSEADDR:
		lock_sock(sk);
		ssock = __mptcp_nmpc_socket(msk);
		if (!ssock) {
			release_sock(sk);
			return -EINVAL;
		}

2767
		ret = sock_setsockopt(ssock, SOL_SOCKET, optname, optval, optlen);
2768 2769 2770 2771 2772 2773 2774 2775 2776 2777
		if (ret == 0) {
			if (optname == SO_REUSEPORT)
				sk->sk_reuseport = ssock->sk->sk_reuseport;
			else if (optname == SO_REUSEADDR)
				sk->sk_reuse = ssock->sk->sk_reuse;
		}
		release_sock(sk);
		return ret;
	}

2778
	return sock_setsockopt(sk->sk_socket, SOL_SOCKET, optname, optval, optlen);
2779 2780
}

2781
static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname,
2782
			       sockptr_t optval, unsigned int optlen)
2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807
{
	struct sock *sk = (struct sock *)msk;
	int ret = -EOPNOTSUPP;
	struct socket *ssock;

	switch (optname) {
	case IPV6_V6ONLY:
		lock_sock(sk);
		ssock = __mptcp_nmpc_socket(msk);
		if (!ssock) {
			release_sock(sk);
			return -EINVAL;
		}

		ret = tcp_setsockopt(ssock->sk, SOL_IPV6, optname, optval, optlen);
		if (ret == 0)
			sk->sk_ipv6only = ssock->sk->sk_ipv6only;

		release_sock(sk);
		break;
	}

	return ret;
}

2808
static int mptcp_setsockopt(struct sock *sk, int level, int optname,
2809
			    sockptr_t optval, unsigned int optlen)
2810 2811
{
	struct mptcp_sock *msk = mptcp_sk(sk);
2812
	struct sock *ssk;
2813 2814 2815

	pr_debug("msk=%p", msk);

2816
	if (level == SOL_SOCKET)
2817
		return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen);
2818

2819
	/* @@ the meaning of setsockopt() when the socket is connected and
2820 2821 2822 2823
	 * there are multiple subflows is not yet defined. It is up to the
	 * MPTCP-level socket to configure the subflows until the subflow
	 * is in TCP fallback, when TCP socket options are passed through
	 * to the one remaining subflow.
2824 2825
	 */
	lock_sock(sk);
2826
	ssk = __mptcp_tcp_fallback(msk);
2827
	release_sock(sk);
2828 2829
	if (ssk)
		return tcp_setsockopt(ssk, level, optname, optval, optlen);
2830

2831 2832 2833
	if (level == SOL_IPV6)
		return mptcp_setsockopt_v6(msk, optname, optval, optlen);

2834
	return -EOPNOTSUPP;
2835 2836 2837
}

static int mptcp_getsockopt(struct sock *sk, int level, int optname,
2838
			    char __user *optval, int __user *option)
2839 2840
{
	struct mptcp_sock *msk = mptcp_sk(sk);
2841
	struct sock *ssk;
2842 2843 2844

	pr_debug("msk=%p", msk);

2845 2846 2847 2848 2849
	/* @@ the meaning of setsockopt() when the socket is connected and
	 * there are multiple subflows is not yet defined. It is up to the
	 * MPTCP-level socket to configure the subflows until the subflow
	 * is in TCP fallback, when socket options are passed through
	 * to the one remaining subflow.
2850 2851
	 */
	lock_sock(sk);
2852
	ssk = __mptcp_tcp_fallback(msk);
2853
	release_sock(sk);
2854 2855
	if (ssk)
		return tcp_getsockopt(ssk, level, optname, optval, option);
2856

2857
	return -EOPNOTSUPP;
2858 2859
}

2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881
void __mptcp_data_acked(struct sock *sk)
{
	if (!sock_owned_by_user(sk))
		__mptcp_clean_una(sk);
	else
		set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags);

	if (mptcp_pending_data_fin_ack(sk))
		mptcp_schedule_work(sk);
}

void __mptcp_wnd_updated(struct sock *sk, struct sock *ssk)
{
	if (!mptcp_send_head(sk))
		return;

	if (!sock_owned_by_user(sk))
		__mptcp_subflow_push_pending(sk, ssk);
	else
		set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags);
}

2882
#define MPTCP_DEFERRED_ALL (TCPF_WRITE_TIMER_DEFERRED)
2883

P
Paolo Abeni 已提交
2884
/* processes deferred events and flush wmem */
2885 2886 2887 2888
static void mptcp_release_cb(struct sock *sk)
{
	unsigned long flags, nflags;

2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907
	/* push_pending may touch wmem_reserved, do it before the later
	 * cleanup
	 */
	if (test_and_clear_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags))
		__mptcp_clean_una(sk);
	if (test_and_clear_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags)) {
		/* mptcp_push_pending() acquires the subflow socket lock
		 *
		 * 1) can't be invoked in atomic scope
		 * 2) must avoid ABBA deadlock with msk socket spinlock: the RX
		 *    datapath acquires the msk socket spinlock while helding
		 *    the subflow socket lock
		 */

		spin_unlock_bh(&sk->sk_lock.slock);
		mptcp_push_pending(sk, 0);
		spin_lock_bh(&sk->sk_lock.slock);
	}

P
Paolo Abeni 已提交
2908 2909
	/* clear any wmem reservation and errors */
	__mptcp_update_wmem(sk);
2910
	__mptcp_update_rmem(sk);
P
Paolo Abeni 已提交
2911

2912 2913 2914 2915 2916 2917 2918
	do {
		flags = sk->sk_tsq_flags;
		if (!(flags & MPTCP_DEFERRED_ALL))
			return;
		nflags = flags & ~MPTCP_DEFERRED_ALL;
	} while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);

2919 2920 2921 2922 2923 2924
	sock_release_ownership(sk);

	if (flags & TCPF_WRITE_TIMER_DEFERRED) {
		mptcp_retransmit_handler(sk);
		__sock_put(sk);
	}
2925 2926
}

P
Paolo Abeni 已提交
2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940
static int mptcp_hash(struct sock *sk)
{
	/* should never be called,
	 * we hash the TCP subflows not the master socket
	 */
	WARN_ON_ONCE(1);
	return 0;
}

static void mptcp_unhash(struct sock *sk)
{
	/* called from sk_common_release(), but nothing to do here */
}

2941
static int mptcp_get_port(struct sock *sk, unsigned short snum)
M
Mat Martineau 已提交
2942 2943
{
	struct mptcp_sock *msk = mptcp_sk(sk);
2944
	struct socket *ssock;
M
Mat Martineau 已提交
2945

2946 2947 2948 2949
	ssock = __mptcp_nmpc_socket(msk);
	pr_debug("msk=%p, subflow=%p", msk, ssock);
	if (WARN_ON_ONCE(!ssock))
		return -EINVAL;
M
Mat Martineau 已提交
2950

2951 2952
	return inet_csk_get_port(ssock->sk, snum);
}
M
Mat Martineau 已提交
2953

2954 2955 2956 2957 2958
void mptcp_finish_connect(struct sock *ssk)
{
	struct mptcp_subflow_context *subflow;
	struct mptcp_sock *msk;
	struct sock *sk;
2959
	u64 ack_seq;
M
Mat Martineau 已提交
2960

2961 2962 2963 2964
	subflow = mptcp_subflow_ctx(ssk);
	sk = subflow->conn;
	msk = mptcp_sk(sk);

2965 2966
	pr_debug("msk=%p, token=%u", sk, subflow->token);

2967 2968
	mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq);
	ack_seq++;
2969 2970
	subflow->map_seq = ack_seq;
	subflow->map_subflow_seq = 1;
2971

2972 2973 2974 2975 2976
	/* the socket is not connected yet, no msk/subflow ops can access/race
	 * accessing the field below
	 */
	WRITE_ONCE(msk->remote_key, subflow->remote_key);
	WRITE_ONCE(msk->local_key, subflow->local_key);
2977
	WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
P
Paolo Abeni 已提交
2978
	WRITE_ONCE(msk->snd_nxt, msk->write_seq);
2979
	WRITE_ONCE(msk->ack_seq, ack_seq);
2980
	WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
2981
	WRITE_ONCE(msk->can_ack, 1);
2982
	WRITE_ONCE(msk->snd_una, msk->write_seq);
2983 2984

	mptcp_pm_new_connection(msk, 0);
2985 2986

	mptcp_rcv_space_init(msk, ssk);
M
Mat Martineau 已提交
2987 2988
}

2989 2990 2991 2992 2993 2994 2995 2996 2997
static void mptcp_sock_graft(struct sock *sk, struct socket *parent)
{
	write_lock_bh(&sk->sk_callback_lock);
	rcu_assign_pointer(sk->sk_wq, &parent->wq);
	sk_set_socket(sk, parent);
	sk->sk_uid = SOCK_INODE(parent)->i_uid;
	write_unlock_bh(&sk->sk_callback_lock);
}

P
Paolo Abeni 已提交
2998
bool mptcp_finish_join(struct sock *ssk)
2999
{
P
Paolo Abeni 已提交
3000
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
3001 3002 3003
	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
	struct sock *parent = (void *)msk;
	struct socket *parent_sock;
3004
	bool ret;
3005 3006 3007 3008

	pr_debug("msk=%p, subflow=%p", msk, subflow);

	/* mptcp socket already closing? */
3009
	if (!mptcp_is_fully_established(parent))
3010 3011 3012 3013 3014
		return false;

	if (!msk->pm.server_side)
		return true;

3015 3016 3017 3018 3019 3020
	if (!mptcp_pm_allow_new_subflow(msk))
		return false;

	/* active connections are already on conn_list, and we can't acquire
	 * msk lock here.
	 * use the join list lock as synchronization point and double-check
P
Paolo Abeni 已提交
3021
	 * msk status to avoid racing with __mptcp_destroy_sock()
3022 3023 3024
	 */
	spin_lock_bh(&msk->join_list_lock);
	ret = inet_sk_state_load(parent) == TCP_ESTABLISHED;
P
Paolo Abeni 已提交
3025
	if (ret && !WARN_ON_ONCE(!list_empty(&subflow->node))) {
3026
		list_add_tail(&subflow->node, &msk->join_list);
P
Paolo Abeni 已提交
3027 3028
		sock_hold(ssk);
	}
3029 3030 3031 3032 3033 3034 3035
	spin_unlock_bh(&msk->join_list_lock);
	if (!ret)
		return false;

	/* attach to msk socket only after we are sure he will deal with us
	 * at close time
	 */
3036
	parent_sock = READ_ONCE(parent->sk_socket);
P
Paolo Abeni 已提交
3037 3038
	if (parent_sock && !ssk->sk_socket)
		mptcp_sock_graft(ssk, parent_sock);
3039
	subflow->map_seq = READ_ONCE(msk->ack_seq);
3040
	return true;
3041 3042
}

M
Mat Martineau 已提交
3043 3044 3045 3046
static struct proto mptcp_prot = {
	.name		= "MPTCP",
	.owner		= THIS_MODULE,
	.init		= mptcp_init_sock,
3047
	.disconnect	= mptcp_disconnect,
M
Mat Martineau 已提交
3048
	.close		= mptcp_close,
3049
	.accept		= mptcp_accept,
3050 3051
	.setsockopt	= mptcp_setsockopt,
	.getsockopt	= mptcp_getsockopt,
M
Mat Martineau 已提交
3052
	.shutdown	= tcp_shutdown,
3053
	.destroy	= mptcp_destroy,
M
Mat Martineau 已提交
3054 3055
	.sendmsg	= mptcp_sendmsg,
	.recvmsg	= mptcp_recvmsg,
3056
	.release_cb	= mptcp_release_cb,
P
Paolo Abeni 已提交
3057 3058
	.hash		= mptcp_hash,
	.unhash		= mptcp_unhash,
3059
	.get_port	= mptcp_get_port,
3060 3061 3062 3063
	.sockets_allocated	= &mptcp_sockets_allocated,
	.memory_allocated	= &tcp_memory_allocated,
	.memory_pressure	= &tcp_memory_pressure,
	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
P
Paolo Abeni 已提交
3064
	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_rmem),
3065
	.sysctl_mem	= sysctl_tcp_mem,
M
Mat Martineau 已提交
3066
	.obj_size	= sizeof(struct mptcp_sock),
P
Paolo Abeni 已提交
3067
	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
M
Mat Martineau 已提交
3068 3069 3070
	.no_autobind	= true,
};

3071 3072 3073 3074
static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
3075
	int err;
3076 3077

	lock_sock(sock->sk);
3078 3079 3080
	ssock = __mptcp_nmpc_socket(msk);
	if (!ssock) {
		err = -EINVAL;
3081 3082 3083 3084
		goto unlock;
	}

	err = ssock->ops->bind(ssock, uaddr, addr_len);
3085 3086
	if (!err)
		mptcp_copy_inaddrs(sock->sk, ssock->sk);
3087 3088 3089 3090 3091 3092

unlock:
	release_sock(sock->sk);
	return err;
}

3093 3094 3095 3096 3097 3098 3099
static void mptcp_subflow_early_fallback(struct mptcp_sock *msk,
					 struct mptcp_subflow_context *subflow)
{
	subflow->request_mptcp = 0;
	__mptcp_do_fallback(msk);
}

3100 3101 3102 3103
static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr,
				int addr_len, int flags)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
P
Paolo Abeni 已提交
3104
	struct mptcp_subflow_context *subflow;
3105 3106 3107 3108
	struct socket *ssock;
	int err;

	lock_sock(sock->sk);
P
Paolo Abeni 已提交
3109 3110 3111 3112 3113 3114 3115 3116
	if (sock->state != SS_UNCONNECTED && msk->subflow) {
		/* pending connection or invalid state, let existing subflow
		 * cope with that
		 */
		ssock = msk->subflow;
		goto do_connect;
	}

3117 3118 3119
	ssock = __mptcp_nmpc_socket(msk);
	if (!ssock) {
		err = -EINVAL;
3120 3121 3122
		goto unlock;
	}

3123 3124
	mptcp_token_destroy(msk);
	inet_sk_state_store(sock->sk, TCP_SYN_SENT);
P
Paolo Abeni 已提交
3125
	subflow = mptcp_subflow_ctx(ssock->sk);
3126 3127 3128 3129 3130
#ifdef CONFIG_TCP_MD5SIG
	/* no MPTCP if MD5SIG is enabled on this socket or we may run out of
	 * TCP option space.
	 */
	if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
3131
		mptcp_subflow_early_fallback(msk, subflow);
3132
#endif
P
Paolo Abeni 已提交
3133
	if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk))
3134
		mptcp_subflow_early_fallback(msk, subflow);
3135

P
Paolo Abeni 已提交
3136
do_connect:
3137
	err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
P
Paolo Abeni 已提交
3138 3139 3140 3141 3142
	sock->state = ssock->state;

	/* on successful connect, the msk state will be moved to established by
	 * subflow_finish_connect()
	 */
3143
	if (!err || err == -EINPROGRESS)
P
Paolo Abeni 已提交
3144 3145 3146
		mptcp_copy_inaddrs(sock->sk, ssock->sk);
	else
		inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
3147 3148 3149 3150 3151 3152

unlock:
	release_sock(sock->sk);
	return err;
}

3153 3154 3155 3156 3157 3158 3159 3160 3161
static int mptcp_listen(struct socket *sock, int backlog)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	pr_debug("msk=%p", msk);

	lock_sock(sock->sk);
3162 3163 3164
	ssock = __mptcp_nmpc_socket(msk);
	if (!ssock) {
		err = -EINVAL;
3165 3166 3167
		goto unlock;
	}

3168 3169
	mptcp_token_destroy(msk);
	inet_sk_state_store(sock->sk, TCP_LISTEN);
3170 3171
	sock_set_flag(sock->sk, SOCK_RCU_FREE);

3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198
	err = ssock->ops->listen(ssock, backlog);
	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
	if (!err)
		mptcp_copy_inaddrs(sock->sk, ssock->sk);

unlock:
	release_sock(sock->sk);
	return err;
}

static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
			       int flags, bool kern)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	pr_debug("msk=%p", msk);

	lock_sock(sock->sk);
	if (sock->sk->sk_state != TCP_LISTEN)
		goto unlock_fail;

	ssock = __mptcp_nmpc_socket(msk);
	if (!ssock)
		goto unlock_fail;

P
Paolo Abeni 已提交
3199
	clear_bit(MPTCP_DATA_READY, &msk->flags);
3200 3201 3202 3203
	sock_hold(ssock->sk);
	release_sock(sock->sk);

	err = ssock->ops->accept(sock, newsock, flags, kern);
3204
	if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) {
3205 3206
		struct mptcp_sock *msk = mptcp_sk(newsock->sk);
		struct mptcp_subflow_context *subflow;
3207 3208 3209 3210 3211 3212
		struct sock *newsk = newsock->sk;
		bool slowpath;

		slowpath = lock_sock_fast(newsk);
		mptcp_copy_inaddrs(newsk, msk->first);
		mptcp_rcv_space_init(msk, msk->first);
3213 3214 3215 3216

		/* set ssk->sk_socket of accept()ed flows to mptcp socket.
		 * This is needed so NOSPACE flag can be set from tcp stack.
		 */
3217
		__mptcp_flush_join_list(msk);
3218
		mptcp_for_each_subflow(msk, subflow) {
3219 3220 3221 3222 3223
			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

			if (!ssk->sk_socket)
				mptcp_sock_graft(ssk, newsock);
		}
3224
		unlock_sock_fast(newsk, slowpath);
3225 3226
	}

P
Paolo Abeni 已提交
3227 3228
	if (inet_csk_listen_poll(ssock->sk))
		set_bit(MPTCP_DATA_READY, &msk->flags);
3229 3230 3231 3232 3233 3234 3235 3236
	sock_put(ssock->sk);
	return err;

unlock_fail:
	release_sock(sock->sk);
	return -EINVAL;
}

P
Paolo Abeni 已提交
3237 3238 3239 3240 3241 3242
static __poll_t mptcp_check_readable(struct mptcp_sock *msk)
{
	return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM :
	       0;
}

3243 3244 3245 3246 3247 3248 3249 3250 3251 3252
static __poll_t mptcp_check_writeable(struct mptcp_sock *msk)
{
	struct sock *sk = (struct sock *)msk;

	if (unlikely(sk->sk_shutdown & SEND_SHUTDOWN))
		return 0;

	if (sk_stream_is_writeable(sk))
		return EPOLLOUT | EPOLLWRNORM;

3253 3254 3255 3256
	set_bit(MPTCP_NOSPACE, &msk->flags);
	smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */
	if (sk_stream_is_writeable(sk))
		return EPOLLOUT | EPOLLWRNORM;
3257

3258
	return 0;
3259 3260
}

3261 3262 3263
static __poll_t mptcp_poll(struct file *file, struct socket *sock,
			   struct poll_table_struct *wait)
{
3264
	struct sock *sk = sock->sk;
3265
	struct mptcp_sock *msk;
3266
	__poll_t mask = 0;
P
Paolo Abeni 已提交
3267
	int state;
3268

3269 3270 3271
	msk = mptcp_sk(sk);
	sock_poll_wait(file, sock, wait);

P
Paolo Abeni 已提交
3272
	state = inet_sk_state_load(sk);
3273
	pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags);
P
Paolo Abeni 已提交
3274 3275 3276 3277 3278
	if (state == TCP_LISTEN)
		return mptcp_check_readable(msk);

	if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) {
		mask |= mptcp_check_readable(msk);
3279
		mask |= mptcp_check_writeable(msk);
P
Paolo Abeni 已提交
3280
	}
3281 3282 3283
	if (sk->sk_shutdown & RCV_SHUTDOWN)
		mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;

3284 3285 3286
	return mask;
}

3287 3288 3289
static int mptcp_shutdown(struct socket *sock, int how)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
P
Paolo Abeni 已提交
3290
	struct sock *sk = sock->sk;
3291 3292 3293 3294
	int ret = 0;

	pr_debug("sk=%p, how=%d", msk, how);

P
Paolo Abeni 已提交
3295
	lock_sock(sk);
3296 3297 3298 3299 3300 3301 3302 3303

	how++;
	if ((how & ~SHUTDOWN_MASK) || !how) {
		ret = -EINVAL;
		goto out_unlock;
	}

	if (sock->state == SS_CONNECTING) {
P
Paolo Abeni 已提交
3304
		if ((1 << sk->sk_state) &
3305 3306 3307 3308 3309 3310
		    (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))
			sock->state = SS_DISCONNECTING;
		else
			sock->state = SS_CONNECTED;
	}

P
Paolo Abeni 已提交
3311 3312 3313
	sk->sk_shutdown |= how;
	if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk))
		__mptcp_wr_shutdown(sk);
3314

3315
	/* Wake up anyone sleeping in poll. */
P
Paolo Abeni 已提交
3316
	sk->sk_state_change(sk);
3317

3318
out_unlock:
P
Paolo Abeni 已提交
3319
	release_sock(sk);
3320 3321 3322 3323

	return ret;
}

3324 3325 3326 3327 3328 3329 3330 3331
static const struct proto_ops mptcp_stream_ops = {
	.family		   = PF_INET,
	.owner		   = THIS_MODULE,
	.release	   = inet_release,
	.bind		   = mptcp_bind,
	.connect	   = mptcp_stream_connect,
	.socketpair	   = sock_no_socketpair,
	.accept		   = mptcp_stream_accept,
3332
	.getname	   = inet_getname,
3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344
	.poll		   = mptcp_poll,
	.ioctl		   = inet_ioctl,
	.gettstamp	   = sock_gettstamp,
	.listen		   = mptcp_listen,
	.shutdown	   = mptcp_shutdown,
	.setsockopt	   = sock_common_setsockopt,
	.getsockopt	   = sock_common_getsockopt,
	.sendmsg	   = inet_sendmsg,
	.recvmsg	   = inet_recvmsg,
	.mmap		   = sock_no_mmap,
	.sendpage	   = inet_sendpage,
};
3345

M
Mat Martineau 已提交
3346 3347 3348 3349
static struct inet_protosw mptcp_protosw = {
	.type		= SOCK_STREAM,
	.protocol	= IPPROTO_MPTCP,
	.prot		= &mptcp_prot,
3350 3351
	.ops		= &mptcp_stream_ops,
	.flags		= INET_PROTOSW_ICSK,
M
Mat Martineau 已提交
3352 3353
};

3354
void __init mptcp_proto_init(void)
M
Mat Martineau 已提交
3355
{
3356 3357
	mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;

3358 3359 3360
	if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL))
		panic("Failed to allocate MPTCP pcpu counter\n");

3361
	mptcp_subflow_init();
3362
	mptcp_pm_init();
P
Paolo Abeni 已提交
3363
	mptcp_token_init();
3364

M
Mat Martineau 已提交
3365 3366 3367 3368
	if (proto_register(&mptcp_prot, 1) != 0)
		panic("Failed to register MPTCP proto.\n");

	inet_register_protosw(&mptcp_protosw);
3369 3370

	BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb));
M
Mat Martineau 已提交
3371 3372 3373
}

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
3374 3375 3376 3377 3378 3379 3380 3381
static const struct proto_ops mptcp_v6_stream_ops = {
	.family		   = PF_INET6,
	.owner		   = THIS_MODULE,
	.release	   = inet6_release,
	.bind		   = mptcp_bind,
	.connect	   = mptcp_stream_connect,
	.socketpair	   = sock_no_socketpair,
	.accept		   = mptcp_stream_accept,
3382
	.getname	   = inet6_getname,
3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394
	.poll		   = mptcp_poll,
	.ioctl		   = inet6_ioctl,
	.gettstamp	   = sock_gettstamp,
	.listen		   = mptcp_listen,
	.shutdown	   = mptcp_shutdown,
	.setsockopt	   = sock_common_setsockopt,
	.getsockopt	   = sock_common_getsockopt,
	.sendmsg	   = inet6_sendmsg,
	.recvmsg	   = inet6_recvmsg,
	.mmap		   = sock_no_mmap,
	.sendpage	   = inet_sendpage,
#ifdef CONFIG_COMPAT
3395
	.compat_ioctl	   = inet6_compat_ioctl,
3396 3397 3398
#endif
};

M
Mat Martineau 已提交
3399 3400
static struct proto mptcp_v6_prot;

3401 3402 3403 3404 3405 3406
static void mptcp_v6_destroy(struct sock *sk)
{
	mptcp_destroy(sk);
	inet6_destroy_sock(sk);
}

M
Mat Martineau 已提交
3407 3408 3409 3410
static struct inet_protosw mptcp_v6_protosw = {
	.type		= SOCK_STREAM,
	.protocol	= IPPROTO_MPTCP,
	.prot		= &mptcp_v6_prot,
3411
	.ops		= &mptcp_v6_stream_ops,
M
Mat Martineau 已提交
3412 3413 3414
	.flags		= INET_PROTOSW_ICSK,
};

3415
int __init mptcp_proto_v6_init(void)
M
Mat Martineau 已提交
3416 3417 3418 3419 3420 3421
{
	int err;

	mptcp_v6_prot = mptcp_prot;
	strcpy(mptcp_v6_prot.name, "MPTCPv6");
	mptcp_v6_prot.slab = NULL;
3422
	mptcp_v6_prot.destroy = mptcp_v6_destroy;
3423
	mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock);
M
Mat Martineau 已提交
3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435

	err = proto_register(&mptcp_v6_prot, 1);
	if (err)
		return err;

	err = inet6_register_protosw(&mptcp_v6_protosw);
	if (err)
		proto_unregister(&mptcp_v6_prot);

	return err;
}
#endif