protocol.c 44.8 KB
Newer Older
M
Mat Martineau 已提交
1 2 3 4 5 6 7 8 9 10 11
// SPDX-License-Identifier: GPL-2.0
/* Multipath TCP
 *
 * Copyright (c) 2017 - 2019, Intel Corporation.
 */

#define pr_fmt(fmt) "MPTCP: " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/netdevice.h>
12 13
#include <linux/sched/signal.h>
#include <linux/atomic.h>
M
Mat Martineau 已提交
14 15 16 17 18
#include <net/sock.h>
#include <net/inet_common.h>
#include <net/inet_hashtables.h>
#include <net/protocol.h>
#include <net/tcp.h>
19 20 21
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
#include <net/transp_v6.h>
#endif
M
Mat Martineau 已提交
22 23 24
#include <net/mptcp.h>
#include "protocol.h"

25 26
#define MPTCP_SAME_STATE TCP_MAX_STATES

27 28 29 30 31 32 33
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
struct mptcp6_sock {
	struct mptcp_sock msk;
	struct ipv6_pinfo np;
};
#endif

34 35 36 37 38 39
struct mptcp_skb_cb {
	u32 offset;
};

#define MPTCP_SKB_CB(__skb)	((struct mptcp_skb_cb *)&((__skb)->cb[0]))

40 41
static struct percpu_counter mptcp_sockets_allocated;

42 43 44 45 46 47
/* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not
 * completed yet or has failed, return the subflow socket.
 * Otherwise return NULL.
 */
static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk)
{
48
	if (!msk->subflow || READ_ONCE(msk->can_ack))
49 50 51 52 53
		return NULL;

	return msk->subflow;
}

54 55 56 57 58 59
static bool __mptcp_needs_tcp_fallback(const struct mptcp_sock *msk)
{
	return msk->first && !sk_is_mptcp(msk->first);
}

static struct socket *__mptcp_tcp_fallback(struct mptcp_sock *msk)
60 61 62
{
	sock_owned_by_me((const struct sock *)msk);

63
	if (likely(!__mptcp_needs_tcp_fallback(msk)))
64 65
		return NULL;

66
	if (msk->subflow) {
67 68
		release_sock((struct sock *)msk);
		return msk->subflow;
69 70
	}

71
	return NULL;
72 73
}

74 75
static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk)
{
76
	return !msk->first;
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
}

static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state)
{
	struct mptcp_subflow_context *subflow;
	struct sock *sk = (struct sock *)msk;
	struct socket *ssock;
	int err;

	ssock = __mptcp_nmpc_socket(msk);
	if (ssock)
		goto set_state;

	if (!__mptcp_can_create_subflow(msk))
		return ERR_PTR(-EINVAL);

	err = mptcp_subflow_create_socket(sk, &ssock);
	if (err)
		return ERR_PTR(err);

97
	msk->first = ssock->sk;
98 99
	msk->subflow = ssock;
	subflow = mptcp_subflow_ctx(ssock->sk);
100
	list_add(&subflow->node, &msk->conn_list);
101 102 103 104 105 106 107 108
	subflow->request_mptcp = 1;

set_state:
	if (state != MPTCP_SAME_STATE)
		inet_sk_state_store(sk, state);
	return ssock;
}

109 110 111 112 113 114 115
static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,
			     struct sk_buff *skb,
			     unsigned int offset, size_t copy_len)
{
	struct sock *sk = (struct sock *)msk;

	__skb_unlink(skb, &ssk->sk_receive_queue);
116
	skb_set_owner_r(skb, sk);
117 118 119 120 121 122 123 124 125 126 127
	__skb_queue_tail(&sk->sk_receive_queue, skb);

	msk->ack_seq += copy_len;
	MPTCP_SKB_CB(skb)->offset = offset;
}

static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
					   struct sock *ssk,
					   unsigned int *bytes)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
128
	struct sock *sk = (struct sock *)msk;
129 130 131 132
	unsigned int moved = 0;
	bool more_data_avail;
	struct tcp_sock *tp;
	bool done = false;
133

134 135 136 137 138 139
	if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
		int rcvbuf = max(ssk->sk_rcvbuf, sk->sk_rcvbuf);

		if (rcvbuf > sk->sk_rcvbuf)
			sk->sk_rcvbuf = rcvbuf;
	}
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182

	tp = tcp_sk(ssk);
	do {
		u32 map_remaining, offset;
		u32 seq = tp->copied_seq;
		struct sk_buff *skb;
		bool fin;

		/* try to move as much data as available */
		map_remaining = subflow->map_data_len -
				mptcp_subflow_get_map_offset(subflow);

		skb = skb_peek(&ssk->sk_receive_queue);
		if (!skb)
			break;

		offset = seq - TCP_SKB_CB(skb)->seq;
		fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
		if (fin) {
			done = true;
			seq++;
		}

		if (offset < skb->len) {
			size_t len = skb->len - offset;

			if (tp->urg_data)
				done = true;

			__mptcp_move_skb(msk, ssk, skb, offset, len);
			seq += len;
			moved += len;

			if (WARN_ON_ONCE(map_remaining < len))
				break;
		} else {
			WARN_ON_ONCE(!fin);
			sk_eat_skb(ssk, skb);
			done = true;
		}

		WRITE_ONCE(tp->copied_seq, seq);
		more_data_avail = mptcp_subflow_data_available(ssk);
183 184 185 186 187

		if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) {
			done = true;
			break;
		}
188 189 190 191 192 193 194
	} while (more_data_avail);

	*bytes = moved;

	return done;
}

195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
/* In most cases we will be able to lock the mptcp socket.  If its already
 * owned, we need to defer to the work queue to avoid ABBA deadlock.
 */
static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)
{
	struct sock *sk = (struct sock *)msk;
	unsigned int moved = 0;

	if (READ_ONCE(sk->sk_lock.owned))
		return false;

	if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock)))
		return false;

	/* must re-check after taking the lock */
	if (!READ_ONCE(sk->sk_lock.owned))
		__mptcp_move_skbs_from_subflow(msk, ssk, &moved);

	spin_unlock_bh(&sk->sk_lock.slock);

	return moved > 0;
}

void mptcp_data_ready(struct sock *sk, struct sock *ssk)
219 220 221 222
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	set_bit(MPTCP_DATA_READY, &msk->flags);
223

224 225 226 227
	if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) &&
	    move_skbs_to_msk(msk, ssk))
		goto wake;

228 229 230 231
	/* don't schedule if mptcp sk is (still) over limit */
	if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf))
		goto wake;

232 233 234 235
	/* mptcp socket is owned, release_cb should retry */
	if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED,
			      &sk->sk_tsq_flags)) {
		sock_hold(sk);
236

237 238 239 240 241
		/* need to try again, its possible release_cb() has already
		 * been called after the test_and_set_bit() above.
		 */
		move_skbs_to_msk(msk, ssk);
	}
242
wake:
243 244 245
	sk->sk_data_ready(sk);
}

246 247 248 249 250 251 252 253 254 255
static void __mptcp_flush_join_list(struct mptcp_sock *msk)
{
	if (likely(list_empty(&msk->join_list)))
		return;

	spin_lock_bh(&msk->join_list_lock);
	list_splice_tail_init(&msk->join_list, &msk->conn_list);
	spin_unlock_bh(&msk->join_list_lock);
}

256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk)
{
	long tout = ssk && inet_csk(ssk)->icsk_pending ?
				      inet_csk(ssk)->icsk_timeout - jiffies : 0;

	if (tout <= 0)
		tout = mptcp_sk(sk)->timer_ival;
	mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;
}

static bool mptcp_timer_pending(struct sock *sk)
{
	return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);
}

static void mptcp_reset_timer(struct sock *sk)
{
	struct inet_connection_sock *icsk = inet_csk(sk);
	unsigned long tout;

	/* should never be called with mptcp level timer cleared */
	tout = READ_ONCE(mptcp_sk(sk)->timer_ival);
	if (WARN_ON_ONCE(!tout))
		tout = TCP_RTO_MIN;
	sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);
}

void mptcp_data_acked(struct sock *sk)
{
	mptcp_reset_timer(sk);
286 287 288 289

	if (!sk_stream_is_writeable(sk) &&
	    schedule_work(&mptcp_sk(sk)->work))
		sock_hold(sk);
290 291 292 293 294 295 296 297 298 299
}

static void mptcp_stop_timer(struct sock *sk)
{
	struct inet_connection_sock *icsk = inet_csk(sk);

	sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
	mptcp_sk(sk)->timer_ival = 0;
}

300 301 302 303 304 305 306 307
static bool mptcp_ext_cache_refill(struct mptcp_sock *msk)
{
	if (!msk->cached_ext)
		msk->cached_ext = __skb_ext_alloc();

	return !!msk->cached_ext;
}

308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow;
	struct sock *sk = (struct sock *)msk;

	sock_owned_by_me(sk);

	mptcp_for_each_subflow(msk, subflow) {
		if (subflow->data_avail)
			return mptcp_subflow_tcp_sock(subflow);
	}

	return NULL;
}

323 324 325
static bool mptcp_skb_can_collapse_to(u64 write_seq,
				      const struct sk_buff *skb,
				      const struct mptcp_ext *mpext)
326 327 328 329 330
{
	if (!tcp_skb_can_collapse_to(skb))
		return false;

	/* can collapse only if MPTCP level sequence is in order */
331
	return mpext && mpext->data_seq + mpext->data_len == write_seq;
332 333
}

334 335 336 337 338 339 340 341
static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk,
				       const struct page_frag *pfrag,
				       const struct mptcp_data_frag *df)
{
	return df && pfrag->page == df->page &&
		df->data_seq + df->data_len == msk->write_seq;
}

342 343 344
static void dfrag_uncharge(struct sock *sk, int len)
{
	sk_mem_uncharge(sk, len);
345
	sk_wmem_queued_add(sk, -len);
346 347 348
}

static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)
349
{
350 351
	int len = dfrag->data_len + dfrag->overhead;

352
	list_del(&dfrag->list);
353
	dfrag_uncharge(sk, len);
354 355 356 357 358 359 360 361
	put_page(dfrag->page);
}

static void mptcp_clean_una(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct mptcp_data_frag *dtmp, *dfrag;
	u64 snd_una = atomic64_read(&msk->snd_una);
362
	bool cleaned = false;
363 364 365 366 367

	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {
		if (after64(dfrag->data_seq + dfrag->data_len, snd_una))
			break;

368 369 370 371
		dfrag_clear(sk, dfrag);
		cleaned = true;
	}

372 373 374 375 376 377 378 379 380 381 382
	dfrag = mptcp_rtx_head(sk);
	if (dfrag && after64(snd_una, dfrag->data_seq)) {
		u64 delta = dfrag->data_seq + dfrag->data_len - snd_una;

		dfrag->data_seq += delta;
		dfrag->data_len -= delta;

		dfrag_uncharge(sk, delta);
		cleaned = true;
	}

383 384
	if (cleaned) {
		sk_mem_reclaim_partial(sk);
385 386 387 388

		/* Only wake up writers if a subflow is ready */
		if (test_bit(MPTCP_SEND_SPACE, &msk->flags))
			sk_stream_write_space(sk);
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
	}
}

/* ensure we get enough memory for the frag hdr, beyond some minimal amount of
 * data
 */
static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
{
	if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag),
					pfrag, sk->sk_allocation)))
		return true;

	sk->sk_prot->enter_memory_pressure(sk);
	sk_stream_moderate_sndbuf(sk);
	return false;
}

static struct mptcp_data_frag *
mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag,
		      int orig_offset)
{
	int offset = ALIGN(orig_offset, sizeof(long));
	struct mptcp_data_frag *dfrag;

	dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);
	dfrag->data_len = 0;
	dfrag->data_seq = msk->write_seq;
	dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);
	dfrag->offset = offset + sizeof(struct mptcp_data_frag);
	dfrag->page = pfrag->page;

	return dfrag;
}

423
static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
424 425
			      struct msghdr *msg, struct mptcp_data_frag *dfrag,
			      long *timeo, int *pmss_now,
426
			      int *ps_goal)
427
{
428 429
	int mss_now, avail_size, size_goal, offset, ret, frag_truesize = 0;
	bool dfrag_collapsed, can_collapse = false;
430 431
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct mptcp_ext *mpext = NULL;
432
	bool retransmission = !!dfrag;
433
	struct sk_buff *skb, *tail;
434
	struct page_frag *pfrag;
435 436
	struct page *page;
	u64 *write_seq;
437 438 439 440
	size_t psize;

	/* use the mptcp page cache so that we can easily move the data
	 * from one substream to another, but do per subflow memory accounting
441 442
	 * Note: pfrag is used only !retransmission, but the compiler if
	 * fooled into a warning if we don't init here
443 444
	 */
	pfrag = sk_page_frag(sk);
445
	while ((!retransmission && !mptcp_page_frag_refill(ssk, pfrag)) ||
446 447 448 449
	       !mptcp_ext_cache_refill(msk)) {
		ret = sk_stream_wait_memory(ssk, timeo);
		if (ret)
			return ret;
450 451 452 453 454 455

		/* if sk_stream_wait_memory() sleeps snd_una can change
		 * significantly, refresh the rtx queue
		 */
		mptcp_clean_una(sk);

456 457
		if (unlikely(__mptcp_needs_tcp_fallback(msk)))
			return 0;
458
	}
459 460 461 462 463 464 465
	if (!retransmission) {
		write_seq = &msk->write_seq;
		page = pfrag->page;
	} else {
		write_seq = &dfrag->data_seq;
		page = dfrag->page;
	}
466 467 468

	/* compute copy limit */
	mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags);
469 470 471 472 473 474 475 476 477 478 479 480 481 482
	*pmss_now = mss_now;
	*ps_goal = size_goal;
	avail_size = size_goal;
	skb = tcp_write_queue_tail(ssk);
	if (skb) {
		mpext = skb_ext_find(skb, SKB_EXT_MPTCP);

		/* Limit the write to the size available in the
		 * current skb, if any, so that we create at most a new skb.
		 * Explicitly tells TCP internals to avoid collapsing on later
		 * queue management operation, to avoid breaking the ext <->
		 * SSN association set here
		 */
		can_collapse = (size_goal - skb->len > 0) &&
483
			      mptcp_skb_can_collapse_to(*write_seq, skb, mpext);
484 485 486 487 488
		if (!can_collapse)
			TCP_SKB_CB(skb)->eor = 1;
		else
			avail_size = size_goal - skb->len;
	}
489

490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
	if (!retransmission) {
		/* reuse tail pfrag, if possible, or carve a new one from the
		 * page allocator
		 */
		dfrag = mptcp_rtx_tail(sk);
		offset = pfrag->offset;
		dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);
		if (!dfrag_collapsed) {
			dfrag = mptcp_carve_data_frag(msk, pfrag, offset);
			offset = dfrag->offset;
			frag_truesize = dfrag->overhead;
		}
		psize = min_t(size_t, pfrag->size - offset, avail_size);

		/* Copy to page */
		pr_debug("left=%zu", msg_data_left(msg));
		psize = copy_page_from_iter(pfrag->page, offset,
					    min_t(size_t, msg_data_left(msg),
						  psize),
					    &msg->msg_iter);
		pr_debug("left=%zu", msg_data_left(msg));
		if (!psize)
			return -EINVAL;

		if (!sk_wmem_schedule(sk, psize + dfrag->overhead))
			return -ENOMEM;
	} else {
517
		offset = dfrag->offset;
518
		psize = min_t(size_t, dfrag->data_len, avail_size);
519
	}
520

521 522
	/* tell the TCP stack to delay the push so that we can safely
	 * access the skb after the sendpages call
523
	 */
524
	ret = do_tcp_sendpages(ssk, page, offset, psize,
525 526 527
			       msg->msg_flags | MSG_SENDPAGE_NOTLAST);
	if (ret <= 0)
		return ret;
528 529

	frag_truesize += ret;
530 531 532
	if (!retransmission) {
		if (unlikely(ret < psize))
			iov_iter_revert(&msg->msg_iter, psize - ret);
533

534 535 536 537 538 539 540 541 542 543 544
		/* send successful, keep track of sent data for mptcp-level
		 * retransmission
		 */
		dfrag->data_len += ret;
		if (!dfrag_collapsed) {
			get_page(dfrag->page);
			list_add_tail(&dfrag->list, &msk->rtx_queue);
			sk_wmem_queued_add(sk, frag_truesize);
		} else {
			sk_wmem_queued_add(sk, ret);
		}
545

546 547 548 549 550
		/* charge data on mptcp rtx queue to the master socket
		 * Note: we charge such data both to sk and ssk
		 */
		sk->sk_forward_alloc -= frag_truesize;
	}
551

552 553 554 555 556 557 558 559 560 561 562 563
	/* if the tail skb extension is still the cached one, collapsing
	 * really happened. Note: we can't check for 'same skb' as the sk_buff
	 * hdr on tail can be transmitted, freed and re-allocated by the
	 * do_tcp_sendpages() call
	 */
	tail = tcp_write_queue_tail(ssk);
	if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) {
		WARN_ON_ONCE(!can_collapse);
		mpext->data_len += ret;
		goto out;
	}

564 565 566 567 568
	skb = tcp_write_queue_tail(ssk);
	mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext);
	msk->cached_ext = NULL;

	memset(mpext, 0, sizeof(*mpext));
569
	mpext->data_seq = *write_seq;
570 571 572 573 574 575 576 577 578
	mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;
	mpext->data_len = ret;
	mpext->use_map = 1;
	mpext->dsn64 = 1;

	pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d",
		 mpext->data_seq, mpext->subflow_seq, mpext->data_len,
		 mpext->dsn64);

579
out:
580 581 582
	if (!retransmission)
		pfrag->offset += frag_truesize;
	*write_seq += ret;
583 584 585 586 587
	mptcp_subflow_ctx(ssk)->rel_write_seq += ret;

	return ret;
}

588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624
static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow;
	struct sock *backup = NULL;

	sock_owned_by_me((const struct sock *)msk);

	mptcp_for_each_subflow(msk, subflow) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

		if (!sk_stream_memory_free(ssk)) {
			struct socket *sock = ssk->sk_socket;

			if (sock) {
				clear_bit(MPTCP_SEND_SPACE, &msk->flags);
				smp_mb__after_atomic();

				/* enables sk->write_space() callbacks */
				set_bit(SOCK_NOSPACE, &sock->flags);
			}

			return NULL;
		}

		if (subflow->backup) {
			if (!backup)
				backup = ssk;

			continue;
		}

		return ssk;
	}

	return backup;
}

625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
static void ssk_check_wmem(struct mptcp_sock *msk, struct sock *ssk)
{
	struct socket *sock;

	if (likely(sk_stream_is_writeable(ssk)))
		return;

	sock = READ_ONCE(ssk->sk_socket);

	if (sock) {
		clear_bit(MPTCP_SEND_SPACE, &msk->flags);
		smp_mb__after_atomic();
		/* set NOSPACE only after clearing SEND_SPACE flag */
		set_bit(SOCK_NOSPACE, &sock->flags);
	}
}

M
Mat Martineau 已提交
642 643
static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
{
644
	int mss_now = 0, size_goal = 0, ret = 0;
M
Mat Martineau 已提交
645
	struct mptcp_sock *msk = mptcp_sk(sk);
646
	struct socket *ssock;
647
	size_t copied = 0;
648
	struct sock *ssk;
649
	long timeo;
M
Mat Martineau 已提交
650 651 652 653

	if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
		return -EOPNOTSUPP;

654
	lock_sock(sk);
655 656 657 658 659 660 661 662 663

	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

	if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
		ret = sk_stream_wait_connect(sk, &timeo);
		if (ret)
			goto out;
	}

664
	ssock = __mptcp_tcp_fallback(msk);
665 666
	if (unlikely(ssock)) {
fallback:
667 668
		pr_debug("fallback passthrough");
		ret = sock_sendmsg(ssock, msg);
669
		return ret >= 0 ? ret + copied : (copied ? copied : ret);
670 671
	}

672 673
	mptcp_clean_una(sk);

674
	__mptcp_flush_join_list(msk);
675 676 677 678 679 680
	ssk = mptcp_subflow_get_send(msk);
	while (!sk_stream_memory_free(sk) || !ssk) {
		ret = sk_stream_wait_memory(sk, &timeo);
		if (ret)
			goto out;

681 682
		mptcp_clean_una(sk);

683 684 685 686 687
		ssk = mptcp_subflow_get_send(msk);
		if (list_empty(&msk->conn_list)) {
			ret = -ENOTCONN;
			goto out;
		}
688 689
	}

690
	pr_debug("conn_list->subflow=%p", ssk);
691

692 693
	lock_sock(ssk);
	while (msg_data_left(msg)) {
694
		ret = mptcp_sendmsg_frag(sk, ssk, msg, NULL, &timeo, &mss_now,
695
					 &size_goal);
696 697
		if (ret < 0)
			break;
698 699 700 701 702
		if (ret == 0 && unlikely(__mptcp_needs_tcp_fallback(msk))) {
			release_sock(ssk);
			ssock = __mptcp_tcp_fallback(msk);
			goto fallback;
		}
703 704 705 706

		copied += ret;
	}

707
	mptcp_set_timeout(sk, ssk);
708
	if (copied) {
709
		ret = copied;
710 711
		tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle,
			 size_goal);
712 713 714 715

		/* start the timer, if it's not pending */
		if (!mptcp_timer_pending(sk))
			mptcp_reset_timer(sk);
716
	}
717

718
	ssk_check_wmem(msk, ssk);
719
	release_sock(ssk);
720
out:
721 722
	release_sock(sk);
	return ret;
M
Mat Martineau 已提交
723 724
}

725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
static void mptcp_wait_data(struct sock *sk, long *timeo)
{
	DEFINE_WAIT_FUNC(wait, woken_wake_function);
	struct mptcp_sock *msk = mptcp_sk(sk);

	add_wait_queue(sk_sleep(sk), &wait);
	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);

	sk_wait_event(sk, timeo,
		      test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait);

	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
	remove_wait_queue(sk_sleep(sk), &wait);
}

740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
				struct msghdr *msg,
				size_t len)
{
	struct sock *sk = (struct sock *)msk;
	struct sk_buff *skb;
	int copied = 0;

	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
		u32 offset = MPTCP_SKB_CB(skb)->offset;
		u32 data_len = skb->len - offset;
		u32 count = min_t(size_t, len - copied, data_len);
		int err;

		err = skb_copy_datagram_msg(skb, offset, msg, count);
		if (unlikely(err < 0)) {
			if (!copied)
				return err;
			break;
		}

		copied += count;

		if (count < data_len) {
			MPTCP_SKB_CB(skb)->offset += count;
			break;
		}

		__skb_unlink(skb, &sk->sk_receive_queue);
		__kfree_skb(skb);

		if (copied >= len)
			break;
	}

	return copied;
}

static bool __mptcp_move_skbs(struct mptcp_sock *msk)
{
	unsigned int moved = 0;
	bool done;

	do {
		struct sock *ssk = mptcp_subflow_recv_lookup(msk);

		if (!ssk)
			break;

		lock_sock(ssk);
		done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
		release_sock(ssk);
	} while (!done);

	return moved > 0;
}

M
Mat Martineau 已提交
797 798 799 800
static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
			 int nonblock, int flags, int *addr_len)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
801 802
	struct socket *ssock;
	int copied = 0;
803 804
	int target;
	long timeo;
M
Mat Martineau 已提交
805 806 807 808

	if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT))
		return -EOPNOTSUPP;

809 810
	lock_sock(sk);
	ssock = __mptcp_tcp_fallback(msk);
811 812
	if (unlikely(ssock)) {
fallback:
813 814 815 816 817 818
		pr_debug("fallback-read subflow=%p",
			 mptcp_subflow_ctx(ssock->sk));
		copied = sock_recvmsg(ssock, msg, flags);
		return copied;
	}

819 820 821 822
	timeo = sock_rcvtimeo(sk, nonblock);

	len = min_t(size_t, len, INT_MAX);
	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
823
	__mptcp_flush_join_list(msk);
824

825
	while (len > (size_t)copied) {
826 827
		int bytes_read;

828 829 830 831 832 833
		bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied);
		if (unlikely(bytes_read < 0)) {
			if (!copied)
				copied = bytes_read;
			goto out_err;
		}
834

835
		copied += bytes_read;
836

837 838 839
		if (skb_queue_empty(&sk->sk_receive_queue) &&
		    __mptcp_move_skbs(msk))
			continue;
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880

		/* only the master socket status is relevant here. The exit
		 * conditions mirror closely tcp_recvmsg()
		 */
		if (copied >= target)
			break;

		if (copied) {
			if (sk->sk_err ||
			    sk->sk_state == TCP_CLOSE ||
			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
			    !timeo ||
			    signal_pending(current))
				break;
		} else {
			if (sk->sk_err) {
				copied = sock_error(sk);
				break;
			}

			if (sk->sk_shutdown & RCV_SHUTDOWN)
				break;

			if (sk->sk_state == TCP_CLOSE) {
				copied = -ENOTCONN;
				break;
			}

			if (!timeo) {
				copied = -EAGAIN;
				break;
			}

			if (signal_pending(current)) {
				copied = sock_intr_errno(timeo);
				break;
			}
		}

		pr_debug("block timeout %ld", timeo);
		mptcp_wait_data(sk, &timeo);
881 882
		if (unlikely(__mptcp_tcp_fallback(msk)))
			goto fallback;
883 884
	}

885 886
	if (skb_queue_empty(&sk->sk_receive_queue)) {
		/* entire backlog drained, clear DATA_READY. */
887
		clear_bit(MPTCP_DATA_READY, &msk->flags);
888

889 890
		/* .. race-breaker: ssk might have gotten new data
		 * after last __mptcp_move_skbs() returned false.
891
		 */
892
		if (unlikely(__mptcp_move_skbs(msk)))
893
			set_bit(MPTCP_DATA_READY, &msk->flags);
894 895 896
	} else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) {
		/* data to read but mptcp_wait_data() cleared DATA_READY */
		set_bit(MPTCP_DATA_READY, &msk->flags);
897
	}
898
out_err:
899
	release_sock(sk);
900 901 902
	return copied;
}

903 904 905 906
static void mptcp_retransmit_handler(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

907
	if (atomic64_read(&msk->snd_una) == msk->write_seq) {
908
		mptcp_stop_timer(sk);
909 910 911 912 913
	} else {
		set_bit(MPTCP_WORK_RTX, &msk->flags);
		if (schedule_work(&msk->work))
			sock_hold(sk);
	}
914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
}

static void mptcp_retransmit_timer(struct timer_list *t)
{
	struct inet_connection_sock *icsk = from_timer(icsk, t,
						       icsk_retransmit_timer);
	struct sock *sk = &icsk->icsk_inet.sk;

	bh_lock_sock(sk);
	if (!sock_owned_by_user(sk)) {
		mptcp_retransmit_handler(sk);
	} else {
		/* delegate our work to tcp_release_cb() */
		if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED,
				      &sk->sk_tsq_flags))
			sock_hold(sk);
	}
	bh_unlock_sock(sk);
	sock_put(sk);
}

935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
/* Find an idle subflow.  Return NULL if there is unacked data at tcp
 * level.
 *
 * A backup subflow is returned only if that is the only kind available.
 */
static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow;
	struct sock *backup = NULL;

	sock_owned_by_me((const struct sock *)msk);

	mptcp_for_each_subflow(msk, subflow) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

		/* still data outstanding at TCP level?  Don't retransmit. */
		if (!tcp_write_queue_empty(ssk))
			return NULL;

		if (subflow->backup) {
			if (!backup)
				backup = ssk;
			continue;
		}

		return ssk;
	}

	return backup;
}

966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
/* subflow sockets can be either outgoing (connect) or incoming
 * (accept).
 *
 * Outgoing subflows use in-kernel sockets.
 * Incoming subflows do not have their own 'struct socket' allocated,
 * so we need to use tcp_close() after detaching them from the mptcp
 * parent socket.
 */
static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
			      struct mptcp_subflow_context *subflow,
			      long timeout)
{
	struct socket *sock = READ_ONCE(ssk->sk_socket);

	list_del(&subflow->node);

	if (sock && sock != sk->sk_socket) {
		/* outgoing subflow */
		sock_release(sock);
	} else {
		/* incoming subflow */
		tcp_close(ssk, timeout);
	}
M
Mat Martineau 已提交
989 990
}

P
Paolo Abeni 已提交
991 992 993 994 995
static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu)
{
	return 0;
}

P
Paolo Abeni 已提交
996 997 998
static void mptcp_worker(struct work_struct *work)
{
	struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work);
999 1000 1001 1002 1003 1004 1005
	struct sock *ssk, *sk = &msk->sk.icsk_inet.sk;
	int orig_len, orig_offset, ret, mss_now = 0, size_goal = 0;
	struct mptcp_data_frag *dfrag;
	u64 orig_write_seq;
	size_t copied = 0;
	struct msghdr msg;
	long timeo = 0;
P
Paolo Abeni 已提交
1006 1007

	lock_sock(sk);
1008
	mptcp_clean_una(sk);
1009
	__mptcp_flush_join_list(msk);
1010
	__mptcp_move_skbs(msk);
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054

	if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags))
		goto unlock;

	dfrag = mptcp_rtx_head(sk);
	if (!dfrag)
		goto unlock;

	ssk = mptcp_subflow_get_retrans(msk);
	if (!ssk)
		goto reset_unlock;

	lock_sock(ssk);

	msg.msg_flags = MSG_DONTWAIT;
	orig_len = dfrag->data_len;
	orig_offset = dfrag->offset;
	orig_write_seq = dfrag->data_seq;
	while (dfrag->data_len > 0) {
		ret = mptcp_sendmsg_frag(sk, ssk, &msg, dfrag, &timeo, &mss_now,
					 &size_goal);
		if (ret < 0)
			break;

		copied += ret;
		dfrag->data_len -= ret;
		dfrag->offset += ret;
	}
	if (copied)
		tcp_push(ssk, msg.msg_flags, mss_now, tcp_sk(ssk)->nonagle,
			 size_goal);

	dfrag->data_seq = orig_write_seq;
	dfrag->offset = orig_offset;
	dfrag->data_len = orig_len;

	mptcp_set_timeout(sk, ssk);
	release_sock(ssk);

reset_unlock:
	if (!mptcp_timer_pending(sk))
		mptcp_reset_timer(sk);

unlock:
P
Paolo Abeni 已提交
1055 1056 1057 1058
	release_sock(sk);
	sock_put(sk);
}

1059
static int __mptcp_init_sock(struct sock *sk)
M
Mat Martineau 已提交
1060
{
1061 1062
	struct mptcp_sock *msk = mptcp_sk(sk);

1063 1064
	spin_lock_init(&msk->join_list_lock);

1065
	INIT_LIST_HEAD(&msk->conn_list);
1066
	INIT_LIST_HEAD(&msk->join_list);
1067
	INIT_LIST_HEAD(&msk->rtx_queue);
1068
	__set_bit(MPTCP_SEND_SPACE, &msk->flags);
P
Paolo Abeni 已提交
1069
	INIT_WORK(&msk->work, mptcp_worker);
1070

1071
	msk->first = NULL;
P
Paolo Abeni 已提交
1072
	inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss;
1073

1074 1075
	mptcp_pm_data_init(msk);

1076 1077 1078
	/* re-use the csk retrans timer for MPTCP-level retrans */
	timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0);

M
Mat Martineau 已提交
1079 1080 1081
	return 0;
}

1082 1083
static int mptcp_init_sock(struct sock *sk)
{
1084 1085 1086 1087 1088
	int ret = __mptcp_init_sock(sk);

	if (ret)
		return ret;

1089
	sk_sockets_allocated_inc(sk);
1090
	sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[2];
1091

1092 1093 1094
	if (!mptcp_is_enabled(sock_net(sk)))
		return -ENOPROTOOPT;

1095 1096 1097 1098 1099 1100 1101 1102
	return 0;
}

static void __mptcp_clear_xmit(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct mptcp_data_frag *dtmp, *dfrag;

1103 1104
	sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer);

1105
	list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)
1106
		dfrag_clear(sk, dfrag);
1107 1108
}

P
Paolo Abeni 已提交
1109 1110 1111 1112 1113 1114 1115 1116
static void mptcp_cancel_work(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	if (cancel_work_sync(&msk->work))
		sock_put(sk);
}

1117 1118
static void mptcp_subflow_shutdown(struct sock *ssk, int how,
				   bool data_fin_tx_enable, u64 data_fin_tx_seq)
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
{
	lock_sock(ssk);

	switch (ssk->sk_state) {
	case TCP_LISTEN:
		if (!(how & RCV_SHUTDOWN))
			break;
		/* fall through */
	case TCP_SYN_SENT:
		tcp_disconnect(ssk, O_NONBLOCK);
		break;
	default:
1131 1132 1133 1134 1135 1136 1137 1138
		if (data_fin_tx_enable) {
			struct mptcp_subflow_context *subflow;

			subflow = mptcp_subflow_ctx(ssk);
			subflow->data_fin_tx_seq = data_fin_tx_seq;
			subflow->data_fin_tx_enable = 1;
		}

1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
		ssk->sk_shutdown |= how;
		tcp_shutdown(ssk, how);
		break;
	}

	/* Wake up anyone sleeping in poll. */
	ssk->sk_state_change(ssk);
	release_sock(ssk);
}

1149
/* Called with msk lock held, releases such lock before returning */
1150
static void mptcp_close(struct sock *sk, long timeout)
M
Mat Martineau 已提交
1151
{
1152
	struct mptcp_subflow_context *subflow, *tmp;
M
Mat Martineau 已提交
1153
	struct mptcp_sock *msk = mptcp_sk(sk);
1154
	LIST_HEAD(conn_list);
1155
	u64 data_fin_tx_seq;
M
Mat Martineau 已提交
1156

1157 1158
	lock_sock(sk);

1159
	mptcp_token_destroy(msk->token);
M
Mat Martineau 已提交
1160 1161
	inet_sk_state_store(sk, TCP_CLOSE);

1162 1163
	__mptcp_flush_join_list(msk);

1164 1165
	list_splice_init(&msk->conn_list, &conn_list);

1166 1167
	data_fin_tx_seq = msk->write_seq;

1168 1169
	__mptcp_clear_xmit(sk);

1170 1171 1172
	release_sock(sk);

	list_for_each_entry_safe(subflow, tmp, &conn_list, node) {
1173 1174
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

1175 1176
		subflow->data_fin_tx_seq = data_fin_tx_seq;
		subflow->data_fin_tx_enable = 1;
1177
		__mptcp_close_ssk(sk, ssk, subflow, timeout);
M
Mat Martineau 已提交
1178 1179
	}

P
Paolo Abeni 已提交
1180
	mptcp_cancel_work(sk);
1181
	mptcp_pm_close(msk);
P
Paolo Abeni 已提交
1182

1183 1184
	__skb_queue_purge(&sk->sk_receive_queue);

1185
	sk_common_release(sk);
M
Mat Martineau 已提交
1186 1187
}

1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)
{
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
	struct ipv6_pinfo *msk6 = inet6_sk(msk);

	msk->sk_v6_daddr = ssk->sk_v6_daddr;
	msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;

	if (msk6 && ssk6) {
		msk6->saddr = ssk6->saddr;
		msk6->flow_label = ssk6->flow_label;
	}
#endif

	inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
	inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
	inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
	inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
	inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
	inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
}

1211 1212 1213 1214 1215
static int mptcp_disconnect(struct sock *sk, int flags)
{
	lock_sock(sk);
	__mptcp_clear_xmit(sk);
	release_sock(sk);
1216
	mptcp_cancel_work(sk);
1217 1218 1219
	return tcp_disconnect(sk, flags);
}

1220 1221 1222 1223 1224 1225 1226 1227 1228
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk)
{
	unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo);

	return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
}
#endif

P
Paolo Abeni 已提交
1229
struct sock *mptcp_sk_clone(const struct sock *sk, struct request_sock *req)
1230
{
P
Paolo Abeni 已提交
1231
	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
1232
	struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC);
P
Paolo Abeni 已提交
1233 1234
	struct mptcp_sock *msk;
	u64 ack_seq;
1235 1236 1237 1238 1239 1240 1241 1242 1243

	if (!nsk)
		return NULL;

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	if (nsk->sk_family == AF_INET6)
		inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk);
#endif

P
Paolo Abeni 已提交
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
	__mptcp_init_sock(nsk);

	msk = mptcp_sk(nsk);
	msk->local_key = subflow_req->local_key;
	msk->token = subflow_req->token;
	msk->subflow = NULL;

	if (unlikely(mptcp_token_new_accept(subflow_req->token, nsk))) {
		bh_unlock_sock(nsk);

		/* we can't call into mptcp_close() here - possible BH context
		 * free the sock directly
		 */
		nsk->sk_prot->destroy(nsk);
		sk_free(nsk);
		return NULL;
	}

	msk->write_seq = subflow_req->idsn + 1;
1263
	atomic64_set(&msk->snd_una, msk->write_seq);
P
Paolo Abeni 已提交
1264 1265 1266 1267 1268 1269 1270
	if (subflow_req->remote_key_valid) {
		msk->can_ack = true;
		msk->remote_key = subflow_req->remote_key;
		mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
		ack_seq++;
		msk->ack_seq = ack_seq;
	}
1271 1272 1273

	/* will be fully established after successful MPC subflow creation */
	inet_sk_state_store(nsk, TCP_SYN_RECV);
P
Paolo Abeni 已提交
1274 1275 1276 1277
	bh_unlock_sock(nsk);

	/* keep a single reference */
	__sock_put(nsk);
1278 1279 1280
	return nsk;
}

1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
				 bool kern)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *listener;
	struct sock *newsk;

	listener = __mptcp_nmpc_socket(msk);
	if (WARN_ON_ONCE(!listener)) {
		*err = -EINVAL;
		return NULL;
	}

	pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk));
	newsk = inet_csk_accept(listener->sk, flags, err, kern);
	if (!newsk)
		return NULL;

	pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk));

	if (sk_is_mptcp(newsk)) {
		struct mptcp_subflow_context *subflow;
		struct sock *new_mptcp_sock;
		struct sock *ssk = newsk;

		subflow = mptcp_subflow_ctx(newsk);
P
Paolo Abeni 已提交
1307
		new_mptcp_sock = subflow->conn;
1308

P
Paolo Abeni 已提交
1309 1310 1311 1312 1313 1314
		/* is_mptcp should be false if subflow->conn is missing, see
		 * subflow_syn_recv_sock()
		 */
		if (WARN_ON_ONCE(!new_mptcp_sock)) {
			tcp_sk(newsk)->is_mptcp = 0;
			return newsk;
1315 1316
		}

P
Paolo Abeni 已提交
1317 1318
		/* acquire the 2nd reference for the owning socket */
		sock_hold(new_mptcp_sock);
1319

P
Paolo Abeni 已提交
1320 1321
		local_bh_disable();
		bh_lock_sock(new_mptcp_sock);
1322
		msk = mptcp_sk(new_mptcp_sock);
1323
		msk->first = newsk;
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335

		newsk = new_mptcp_sock;
		mptcp_copy_inaddrs(newsk, ssk);
		list_add(&subflow->node, &msk->conn_list);

		bh_unlock_sock(new_mptcp_sock);
		local_bh_enable();
	}

	return newsk;
}

1336 1337
static void mptcp_destroy(struct sock *sk)
{
1338 1339 1340 1341
	struct mptcp_sock *msk = mptcp_sk(sk);

	if (msk->cached_ext)
		__skb_ext_put(msk->cached_ext);
1342 1343

	sk_sockets_allocated_dec(sk);
1344 1345
}

1346
static int mptcp_setsockopt(struct sock *sk, int level, int optname,
1347
			    char __user *optval, unsigned int optlen)
1348 1349 1350 1351 1352 1353 1354
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *ssock;

	pr_debug("msk=%p", msk);

	/* @@ the meaning of setsockopt() when the socket is connected and
1355 1356 1357 1358
	 * there are multiple subflows is not yet defined. It is up to the
	 * MPTCP-level socket to configure the subflows until the subflow
	 * is in TCP fallback, when TCP socket options are passed through
	 * to the one remaining subflow.
1359 1360
	 */
	lock_sock(sk);
1361 1362 1363 1364
	ssock = __mptcp_tcp_fallback(msk);
	if (ssock)
		return tcp_setsockopt(ssock->sk, level, optname, optval,
				      optlen);
1365

1366 1367
	release_sock(sk);

1368
	return -EOPNOTSUPP;
1369 1370 1371
}

static int mptcp_getsockopt(struct sock *sk, int level, int optname,
1372
			    char __user *optval, int __user *option)
1373 1374 1375 1376 1377 1378
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *ssock;

	pr_debug("msk=%p", msk);

1379 1380 1381 1382 1383
	/* @@ the meaning of setsockopt() when the socket is connected and
	 * there are multiple subflows is not yet defined. It is up to the
	 * MPTCP-level socket to configure the subflows until the subflow
	 * is in TCP fallback, when socket options are passed through
	 * to the one remaining subflow.
1384 1385
	 */
	lock_sock(sk);
1386 1387 1388 1389
	ssock = __mptcp_tcp_fallback(msk);
	if (ssock)
		return tcp_getsockopt(ssock->sk, level, optname, optval,
				      option);
1390

1391 1392
	release_sock(sk);

1393
	return -EOPNOTSUPP;
1394 1395
}

1396 1397
#define MPTCP_DEFERRED_ALL (TCPF_DELACK_TIMER_DEFERRED | \
			    TCPF_WRITE_TIMER_DEFERRED)
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412

/* this is very alike tcp_release_cb() but we must handle differently a
 * different set of events
 */
static void mptcp_release_cb(struct sock *sk)
{
	unsigned long flags, nflags;

	do {
		flags = sk->sk_tsq_flags;
		if (!(flags & MPTCP_DEFERRED_ALL))
			return;
		nflags = flags & ~MPTCP_DEFERRED_ALL;
	} while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);

1413 1414
	sock_release_ownership(sk);

1415 1416 1417 1418 1419 1420 1421 1422
	if (flags & TCPF_DELACK_TIMER_DEFERRED) {
		struct mptcp_sock *msk = mptcp_sk(sk);
		struct sock *ssk;

		ssk = mptcp_subflow_recv_lookup(msk);
		if (!ssk || !schedule_work(&msk->work))
			__sock_put(sk);
	}
1423 1424 1425 1426 1427

	if (flags & TCPF_WRITE_TIMER_DEFERRED) {
		mptcp_retransmit_handler(sk);
		__sock_put(sk);
	}
1428 1429
}

1430
static int mptcp_get_port(struct sock *sk, unsigned short snum)
M
Mat Martineau 已提交
1431 1432
{
	struct mptcp_sock *msk = mptcp_sk(sk);
1433
	struct socket *ssock;
M
Mat Martineau 已提交
1434

1435 1436 1437 1438
	ssock = __mptcp_nmpc_socket(msk);
	pr_debug("msk=%p, subflow=%p", msk, ssock);
	if (WARN_ON_ONCE(!ssock))
		return -EINVAL;
M
Mat Martineau 已提交
1439

1440 1441
	return inet_csk_get_port(ssock->sk, snum);
}
M
Mat Martineau 已提交
1442

1443 1444 1445 1446 1447
void mptcp_finish_connect(struct sock *ssk)
{
	struct mptcp_subflow_context *subflow;
	struct mptcp_sock *msk;
	struct sock *sk;
1448
	u64 ack_seq;
M
Mat Martineau 已提交
1449

1450
	subflow = mptcp_subflow_ctx(ssk);
M
Mat Martineau 已提交
1451

1452 1453 1454 1455 1456 1457
	if (!subflow->mp_capable)
		return;

	sk = subflow->conn;
	msk = mptcp_sk(sk);

1458 1459
	pr_debug("msk=%p, token=%u", sk, subflow->token);

1460 1461
	mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq);
	ack_seq++;
1462 1463
	subflow->map_seq = ack_seq;
	subflow->map_subflow_seq = 1;
1464 1465
	subflow->rel_write_seq = 1;

1466 1467 1468 1469 1470
	/* the socket is not connected yet, no msk/subflow ops can access/race
	 * accessing the field below
	 */
	WRITE_ONCE(msk->remote_key, subflow->remote_key);
	WRITE_ONCE(msk->local_key, subflow->local_key);
1471
	WRITE_ONCE(msk->token, subflow->token);
1472 1473
	WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
	WRITE_ONCE(msk->ack_seq, ack_seq);
1474
	WRITE_ONCE(msk->can_ack, 1);
1475
	atomic64_set(&msk->snd_una, msk->write_seq);
1476 1477

	mptcp_pm_new_connection(msk, 0);
M
Mat Martineau 已提交
1478 1479
}

1480 1481 1482 1483 1484 1485 1486 1487 1488
static void mptcp_sock_graft(struct sock *sk, struct socket *parent)
{
	write_lock_bh(&sk->sk_callback_lock);
	rcu_assign_pointer(sk->sk_wq, &parent->wq);
	sk_set_socket(sk, parent);
	sk->sk_uid = SOCK_INODE(parent)->i_uid;
	write_unlock_bh(&sk->sk_callback_lock);
}

1489 1490 1491 1492 1493 1494
bool mptcp_finish_join(struct sock *sk)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
	struct sock *parent = (void *)msk;
	struct socket *parent_sock;
1495
	bool ret;
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510

	pr_debug("msk=%p, subflow=%p", msk, subflow);

	/* mptcp socket already closing? */
	if (inet_sk_state_load(parent) != TCP_ESTABLISHED)
		return false;

	if (!msk->pm.server_side)
		return true;

	/* passive connection, attach to msk socket */
	parent_sock = READ_ONCE(parent->sk_socket);
	if (parent_sock && !sk->sk_socket)
		mptcp_sock_graft(sk, parent_sock);

1511 1512 1513 1514 1515 1516 1517 1518 1519
	ret = mptcp_pm_allow_new_subflow(msk);
	if (ret) {
		/* active connections are already on conn_list */
		spin_lock_bh(&msk->join_list_lock);
		if (!WARN_ON_ONCE(!list_empty(&subflow->node)))
			list_add_tail(&subflow->node, &msk->join_list);
		spin_unlock_bh(&msk->join_list_lock);
	}
	return ret;
1520 1521 1522 1523 1524 1525 1526 1527 1528
}

bool mptcp_sk_is_subflow(const struct sock *sk)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);

	return subflow->mp_join == 1;
}

1529 1530 1531 1532 1533 1534 1535
static bool mptcp_memory_free(const struct sock *sk, int wake)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true;
}

M
Mat Martineau 已提交
1536 1537 1538 1539
static struct proto mptcp_prot = {
	.name		= "MPTCP",
	.owner		= THIS_MODULE,
	.init		= mptcp_init_sock,
1540
	.disconnect	= mptcp_disconnect,
M
Mat Martineau 已提交
1541
	.close		= mptcp_close,
1542
	.accept		= mptcp_accept,
1543 1544
	.setsockopt	= mptcp_setsockopt,
	.getsockopt	= mptcp_getsockopt,
M
Mat Martineau 已提交
1545
	.shutdown	= tcp_shutdown,
1546
	.destroy	= mptcp_destroy,
M
Mat Martineau 已提交
1547 1548
	.sendmsg	= mptcp_sendmsg,
	.recvmsg	= mptcp_recvmsg,
1549
	.release_cb	= mptcp_release_cb,
M
Mat Martineau 已提交
1550 1551
	.hash		= inet_hash,
	.unhash		= inet_unhash,
1552
	.get_port	= mptcp_get_port,
1553 1554 1555
	.sockets_allocated	= &mptcp_sockets_allocated,
	.memory_allocated	= &tcp_memory_allocated,
	.memory_pressure	= &tcp_memory_pressure,
1556
	.stream_memory_free	= mptcp_memory_free,
1557 1558
	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
	.sysctl_mem	= sysctl_tcp_mem,
M
Mat Martineau 已提交
1559 1560 1561 1562
	.obj_size	= sizeof(struct mptcp_sock),
	.no_autobind	= true,
};

1563 1564 1565 1566
static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
1567
	int err;
1568 1569 1570 1571 1572 1573 1574 1575 1576

	lock_sock(sock->sk);
	ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE);
	if (IS_ERR(ssock)) {
		err = PTR_ERR(ssock);
		goto unlock;
	}

	err = ssock->ops->bind(ssock, uaddr, addr_len);
1577 1578
	if (!err)
		mptcp_copy_inaddrs(sock->sk, ssock->sk);
1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598

unlock:
	release_sock(sock->sk);
	return err;
}

static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr,
				int addr_len, int flags)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	lock_sock(sock->sk);
	ssock = __mptcp_socket_create(msk, TCP_SYN_SENT);
	if (IS_ERR(ssock)) {
		err = PTR_ERR(ssock);
		goto unlock;
	}

1599 1600 1601 1602 1603 1604 1605 1606
#ifdef CONFIG_TCP_MD5SIG
	/* no MPTCP if MD5SIG is enabled on this socket or we may run out of
	 * TCP option space.
	 */
	if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
		mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0;
#endif

1607 1608
	err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
1609
	mptcp_copy_inaddrs(sock->sk, ssock->sk);
1610 1611 1612 1613 1614 1615

unlock:
	release_sock(sock->sk);
	return err;
}

1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr,
			    int peer)
{
	if (sock->sk->sk_prot == &tcp_prot) {
		/* we are being invoked from __sys_accept4, after
		 * mptcp_accept() has just accepted a non-mp-capable
		 * flow: sk is a tcp_sk, not an mptcp one.
		 *
		 * Hand the socket over to tcp so all further socket ops
		 * bypass mptcp.
		 */
		sock->ops = &inet_stream_ops;
	}

	return inet_getname(sock, uaddr, peer);
}

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr,
			    int peer)
{
	if (sock->sk->sk_prot == &tcpv6_prot) {
		/* we are being invoked from __sys_accept4 after
		 * mptcp_accept() has accepted a non-mp-capable
		 * subflow: sk is a tcp_sk, not mptcp.
		 *
		 * Hand the socket over to tcp so all further
		 * socket ops bypass mptcp.
		 */
		sock->ops = &inet6_stream_ops;
	}

	return inet6_getname(sock, uaddr, peer);
}
#endif

static int mptcp_listen(struct socket *sock, int backlog)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	pr_debug("msk=%p", msk);

	lock_sock(sock->sk);
	ssock = __mptcp_socket_create(msk, TCP_LISTEN);
	if (IS_ERR(ssock)) {
		err = PTR_ERR(ssock);
		goto unlock;
	}

	err = ssock->ops->listen(ssock, backlog);
	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
	if (!err)
		mptcp_copy_inaddrs(sock->sk, ssock->sk);

unlock:
	release_sock(sock->sk);
	return err;
}

static bool is_tcp_proto(const struct proto *p)
{
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	return p == &tcp_prot || p == &tcpv6_prot;
#else
	return p == &tcp_prot;
#endif
}

static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
			       int flags, bool kern)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	pr_debug("msk=%p", msk);

	lock_sock(sock->sk);
	if (sock->sk->sk_state != TCP_LISTEN)
		goto unlock_fail;

	ssock = __mptcp_nmpc_socket(msk);
	if (!ssock)
		goto unlock_fail;

	sock_hold(ssock->sk);
	release_sock(sock->sk);

	err = ssock->ops->accept(sock, newsock, flags, kern);
	if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) {
		struct mptcp_sock *msk = mptcp_sk(newsock->sk);
		struct mptcp_subflow_context *subflow;

		/* set ssk->sk_socket of accept()ed flows to mptcp socket.
		 * This is needed so NOSPACE flag can be set from tcp stack.
		 */
1714
		__mptcp_flush_join_list(msk);
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
		list_for_each_entry(subflow, &msk->conn_list, node) {
			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

			if (!ssk->sk_socket)
				mptcp_sock_graft(ssk, newsock);
		}
	}

	sock_put(ssock->sk);
	return err;

unlock_fail:
	release_sock(sock->sk);
	return -EINVAL;
}

1731 1732 1733
static __poll_t mptcp_poll(struct file *file, struct socket *sock,
			   struct poll_table_struct *wait)
{
1734
	struct sock *sk = sock->sk;
1735
	struct mptcp_sock *msk;
1736
	struct socket *ssock;
1737 1738
	__poll_t mask = 0;

1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750
	msk = mptcp_sk(sk);
	lock_sock(sk);
	ssock = __mptcp_nmpc_socket(msk);
	if (ssock) {
		mask = ssock->ops->poll(file, ssock, wait);
		release_sock(sk);
		return mask;
	}

	release_sock(sk);
	sock_poll_wait(file, sock, wait);
	lock_sock(sk);
1751 1752 1753
	ssock = __mptcp_tcp_fallback(msk);
	if (unlikely(ssock))
		return ssock->ops->poll(file, ssock, NULL);
1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764

	if (test_bit(MPTCP_DATA_READY, &msk->flags))
		mask = EPOLLIN | EPOLLRDNORM;
	if (sk_stream_is_writeable(sk) &&
	    test_bit(MPTCP_SEND_SPACE, &msk->flags))
		mask |= EPOLLOUT | EPOLLWRNORM;
	if (sk->sk_shutdown & RCV_SHUTDOWN)
		mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;

	release_sock(sk);

1765 1766 1767
	return mask;
}

1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
static int mptcp_shutdown(struct socket *sock, int how)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct mptcp_subflow_context *subflow;
	int ret = 0;

	pr_debug("sk=%p, how=%d", msk, how);

	lock_sock(sock->sk);

	if (how == SHUT_WR || how == SHUT_RDWR)
		inet_sk_state_store(sock->sk, TCP_FIN_WAIT1);

	how++;

	if ((how & ~SHUTDOWN_MASK) || !how) {
		ret = -EINVAL;
		goto out_unlock;
	}

	if (sock->state == SS_CONNECTING) {
		if ((1 << sock->sk->sk_state) &
		    (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))
			sock->state = SS_DISCONNECTING;
		else
			sock->state = SS_CONNECTED;
	}

1796
	__mptcp_flush_join_list(msk);
1797 1798 1799
	mptcp_for_each_subflow(msk, subflow) {
		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);

1800
		mptcp_subflow_shutdown(tcp_sk, how, 1, msk->write_seq);
1801 1802 1803 1804 1805 1806 1807 1808
	}

out_unlock:
	release_sock(sock->sk);

	return ret;
}

1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
static const struct proto_ops mptcp_stream_ops = {
	.family		   = PF_INET,
	.owner		   = THIS_MODULE,
	.release	   = inet_release,
	.bind		   = mptcp_bind,
	.connect	   = mptcp_stream_connect,
	.socketpair	   = sock_no_socketpair,
	.accept		   = mptcp_stream_accept,
	.getname	   = mptcp_v4_getname,
	.poll		   = mptcp_poll,
	.ioctl		   = inet_ioctl,
	.gettstamp	   = sock_gettstamp,
	.listen		   = mptcp_listen,
	.shutdown	   = mptcp_shutdown,
	.setsockopt	   = sock_common_setsockopt,
	.getsockopt	   = sock_common_getsockopt,
	.sendmsg	   = inet_sendmsg,
	.recvmsg	   = inet_recvmsg,
	.mmap		   = sock_no_mmap,
	.sendpage	   = inet_sendpage,
#ifdef CONFIG_COMPAT
	.compat_setsockopt = compat_sock_common_setsockopt,
	.compat_getsockopt = compat_sock_common_getsockopt,
#endif
};
1834

M
Mat Martineau 已提交
1835 1836 1837 1838
static struct inet_protosw mptcp_protosw = {
	.type		= SOCK_STREAM,
	.protocol	= IPPROTO_MPTCP,
	.prot		= &mptcp_prot,
1839 1840
	.ops		= &mptcp_stream_ops,
	.flags		= INET_PROTOSW_ICSK,
M
Mat Martineau 已提交
1841 1842
};

1843
void mptcp_proto_init(void)
M
Mat Martineau 已提交
1844
{
1845 1846
	mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;

1847 1848 1849
	if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL))
		panic("Failed to allocate MPTCP pcpu counter\n");

1850
	mptcp_subflow_init();
1851
	mptcp_pm_init();
1852

M
Mat Martineau 已提交
1853 1854 1855 1856
	if (proto_register(&mptcp_prot, 1) != 0)
		panic("Failed to register MPTCP proto.\n");

	inet_register_protosw(&mptcp_protosw);
1857 1858

	BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb));
M
Mat Martineau 已提交
1859 1860 1861
}

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887
static const struct proto_ops mptcp_v6_stream_ops = {
	.family		   = PF_INET6,
	.owner		   = THIS_MODULE,
	.release	   = inet6_release,
	.bind		   = mptcp_bind,
	.connect	   = mptcp_stream_connect,
	.socketpair	   = sock_no_socketpair,
	.accept		   = mptcp_stream_accept,
	.getname	   = mptcp_v6_getname,
	.poll		   = mptcp_poll,
	.ioctl		   = inet6_ioctl,
	.gettstamp	   = sock_gettstamp,
	.listen		   = mptcp_listen,
	.shutdown	   = mptcp_shutdown,
	.setsockopt	   = sock_common_setsockopt,
	.getsockopt	   = sock_common_getsockopt,
	.sendmsg	   = inet6_sendmsg,
	.recvmsg	   = inet6_recvmsg,
	.mmap		   = sock_no_mmap,
	.sendpage	   = inet_sendpage,
#ifdef CONFIG_COMPAT
	.compat_setsockopt = compat_sock_common_setsockopt,
	.compat_getsockopt = compat_sock_common_getsockopt,
#endif
};

M
Mat Martineau 已提交
1888 1889
static struct proto mptcp_v6_prot;

1890 1891 1892 1893 1894 1895
static void mptcp_v6_destroy(struct sock *sk)
{
	mptcp_destroy(sk);
	inet6_destroy_sock(sk);
}

M
Mat Martineau 已提交
1896 1897 1898 1899
static struct inet_protosw mptcp_v6_protosw = {
	.type		= SOCK_STREAM,
	.protocol	= IPPROTO_MPTCP,
	.prot		= &mptcp_v6_prot,
1900
	.ops		= &mptcp_v6_stream_ops,
M
Mat Martineau 已提交
1901 1902 1903
	.flags		= INET_PROTOSW_ICSK,
};

1904
int mptcp_proto_v6_init(void)
M
Mat Martineau 已提交
1905 1906 1907 1908 1909 1910
{
	int err;

	mptcp_v6_prot = mptcp_prot;
	strcpy(mptcp_v6_prot.name, "MPTCPv6");
	mptcp_v6_prot.slab = NULL;
1911
	mptcp_v6_prot.destroy = mptcp_v6_destroy;
1912
	mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock);
M
Mat Martineau 已提交
1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924

	err = proto_register(&mptcp_v6_prot, 1);
	if (err)
		return err;

	err = inet6_register_protosw(&mptcp_v6_protosw);
	if (err)
		proto_unregister(&mptcp_v6_prot);

	return err;
}
#endif