ip6_output.c 46.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 *	IPv6 output functions
3
 *	Linux INET6 implementation
L
Linus Torvalds 已提交
4 5
 *
 *	Authors:
6
 *	Pedro Roque		<roque@di.fc.ul.pt>
L
Linus Torvalds 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *
 *	Based on linux/net/ipv4/ip_output.c
 *
 *	This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 *
 *	Changes:
 *	A.N.Kuznetsov	:	airthmetics in fragmentation.
 *				extension headers are implemented.
 *				route changes now work.
 *				ip6_forward does not confuse sniffers.
 *				etc.
 *
 *      H. von Brand    :       Added missing #include <linux/string.h>
23
 *	Imran Patel	:	frag id should be in NBO
L
Linus Torvalds 已提交
24 25 26 27 28 29
 *      Kazunori MIYAZAWA @USAGI
 *			:       add ip6_append_data and related functions
 *				for datagram xmit
 */

#include <linux/errno.h>
H
Herbert Xu 已提交
30
#include <linux/kernel.h>
L
Linus Torvalds 已提交
31 32 33 34 35 36 37 38
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/in6.h>
#include <linux/tcp.h>
#include <linux/route.h>
39
#include <linux/module.h>
40
#include <linux/slab.h>
L
Linus Torvalds 已提交
41

42
#include <linux/bpf-cgroup.h>
L
Linus Torvalds 已提交
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
#include <linux/netfilter.h>
#include <linux/netfilter_ipv6.h>

#include <net/sock.h>
#include <net/snmp.h>

#include <net/ipv6.h>
#include <net/ndisc.h>
#include <net/protocol.h>
#include <net/ip6_route.h>
#include <net/addrconf.h>
#include <net/rawv6.h>
#include <net/icmp.h>
#include <net/xfrm.h>
#include <net/checksum.h>
58
#include <linux/mroute6.h>
D
David Ahern 已提交
59
#include <net/l3mdev.h>
60
#include <net/lwtunnel.h>
L
Linus Torvalds 已提交
61

62
static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
63
{
E
Eric Dumazet 已提交
64
	struct dst_entry *dst = skb_dst(skb);
L
Linus Torvalds 已提交
65
	struct net_device *dev = dst->dev;
66
	struct neighbour *neigh;
67 68
	struct in6_addr *nexthop;
	int ret;
L
Linus Torvalds 已提交
69 70 71 72

	skb->protocol = htons(ETH_P_IPV6);
	skb->dev = dev;

73
	if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
E
Eric Dumazet 已提交
74
		struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
L
Linus Torvalds 已提交
75

76
		if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) &&
77
		    ((mroute6_socket(net, skb) &&
78
		     !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
79 80
		     ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr,
					 &ipv6_hdr(skb)->saddr))) {
L
Linus Torvalds 已提交
81 82 83 84 85 86
			struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);

			/* Do not check for IFF_ALLMULTI; multicast routing
			   is not supported in any case.
			 */
			if (newskb)
87
				NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
88
					net, sk, newskb, NULL, newskb->dev,
89
					dev_loopback_xmit);
L
Linus Torvalds 已提交
90

91
			if (ipv6_hdr(skb)->hop_limit == 0) {
92
				IP6_INC_STATS(net, idev,
93
					      IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
94 95 96 97 98
				kfree_skb(skb);
				return 0;
			}
		}

99
		IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len);
100 101 102 103 104 105 106

		if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <=
		    IPV6_ADDR_SCOPE_NODELOCAL &&
		    !(dev->flags & IFF_LOOPBACK)) {
			kfree_skb(skb);
			return 0;
		}
L
Linus Torvalds 已提交
107 108
	}

109 110 111 112 113 114 115
	if (lwtunnel_xmit_redirect(dst->lwtstate)) {
		int res = lwtunnel_xmit(skb);

		if (res < 0 || res == LWTUNNEL_XMIT_DONE)
			return res;
	}

116
	rcu_read_lock_bh();
117
	nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr);
118 119 120 121
	neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
	if (unlikely(!neigh))
		neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
	if (!IS_ERR(neigh)) {
122
		sock_confirm_neigh(skb, neigh);
123 124 125 126 127
		ret = dst_neigh_output(dst, neigh, skb);
		rcu_read_unlock_bh();
		return ret;
	}
	rcu_read_unlock_bh();
128

129
	IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
130 131
	kfree_skb(skb);
	return -EINVAL;
L
Linus Torvalds 已提交
132 133
}

134
static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
135
{
136 137 138 139 140 141 142 143
	int ret;

	ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
	if (ret) {
		kfree_skb(skb);
		return ret;
	}

144
	if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) ||
145 146
	    dst_allfrag(skb_dst(skb)) ||
	    (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))
147
		return ip6_fragment(net, sk, skb, ip6_finish_output2);
148
	else
149
		return ip6_finish_output2(net, sk, skb);
150 151
}

E
Eric W. Biederman 已提交
152
int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
153
{
154
	struct net_device *dev = skb_dst(skb)->dev;
E
Eric Dumazet 已提交
155
	struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
156

157
	if (unlikely(idev->cnf.disable_ipv6)) {
158
		IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
159 160 161 162
		kfree_skb(skb);
		return 0;
	}

163 164
	return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING,
			    net, sk, skb, NULL, dev,
165 166
			    ip6_finish_output,
			    !(IP6CB(skb)->flags & IP6SKB_REROUTED));
L
Linus Torvalds 已提交
167 168 169
}

/*
170 171 172 173
 * xmit an sk_buff (used by TCP, SCTP and DCCP)
 * Note : socket lock is not held for SYNACK packets, but might be modified
 * by calls to skb_set_owner_w() and ipv6_local_error(),
 * which are using proper atomic operations or spinlocks.
L
Linus Torvalds 已提交
174
 */
175
int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
176
	     __u32 mark, struct ipv6_txoptions *opt, int tclass)
L
Linus Torvalds 已提交
177
{
178
	struct net *net = sock_net(sk);
179
	const struct ipv6_pinfo *np = inet6_sk(sk);
180
	struct in6_addr *first_hop = &fl6->daddr;
E
Eric Dumazet 已提交
181
	struct dst_entry *dst = skb_dst(skb);
L
Linus Torvalds 已提交
182
	struct ipv6hdr *hdr;
183
	u8  proto = fl6->flowi6_proto;
L
Linus Torvalds 已提交
184
	int seg_len = skb->len;
G
Gerrit Renker 已提交
185
	int hlimit = -1;
L
Linus Torvalds 已提交
186 187 188
	u32 mtu;

	if (opt) {
189
		unsigned int head_room;
L
Linus Torvalds 已提交
190 191 192 193 194 195 196 197 198 199

		/* First: exthdrs may take lots of space (~8K for now)
		   MAX_HEADER is not enough.
		 */
		head_room = opt->opt_nflen + opt->opt_flen;
		seg_len += head_room;
		head_room += sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev);

		if (skb_headroom(skb) < head_room) {
			struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room);
200
			if (!skb2) {
E
Eric Dumazet 已提交
201
				IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
202 203
					      IPSTATS_MIB_OUTDISCARDS);
				kfree_skb(skb);
L
Linus Torvalds 已提交
204 205
				return -ENOBUFS;
			}
206
			consume_skb(skb);
207
			skb = skb2;
208 209 210 211
			/* skb_set_owner_w() changes sk->sk_wmem_alloc atomically,
			 * it is safe to call in our context (socket lock not held)
			 */
			skb_set_owner_w(skb, (struct sock *)sk);
L
Linus Torvalds 已提交
212 213 214 215
		}
		if (opt->opt_flen)
			ipv6_push_frag_opts(skb, opt, &proto);
		if (opt->opt_nflen)
216 217
			ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop,
					     &fl6->saddr);
L
Linus Torvalds 已提交
218 219
	}

220 221
	skb_push(skb, sizeof(struct ipv6hdr));
	skb_reset_network_header(skb);
222
	hdr = ipv6_hdr(skb);
L
Linus Torvalds 已提交
223 224 225 226

	/*
	 *	Fill in the IPv6 header
	 */
227
	if (np)
L
Linus Torvalds 已提交
228 229
		hlimit = np->hop_limit;
	if (hlimit < 0)
230
		hlimit = ip6_dst_hoplimit(dst);
L
Linus Torvalds 已提交
231

232
	ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel,
233
						     np->autoflowlabel, fl6));
234

L
Linus Torvalds 已提交
235 236 237 238
	hdr->payload_len = htons(seg_len);
	hdr->nexthdr = proto;
	hdr->hop_limit = hlimit;

A
Alexey Dobriyan 已提交
239 240
	hdr->saddr = fl6->saddr;
	hdr->daddr = *first_hop;
L
Linus Torvalds 已提交
241

242
	skb->protocol = htons(ETH_P_IPV6);
243
	skb->priority = sk->sk_priority;
244
	skb->mark = mark;
245

L
Linus Torvalds 已提交
246
	mtu = dst_mtu(dst);
W
WANG Cong 已提交
247
	if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) {
E
Eric Dumazet 已提交
248
		IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)),
249
			      IPSTATS_MIB_OUT, skb->len);
250 251 252 253 254 255 256 257

		/* if egress device is enslaved to an L3 master device pass the
		 * skb to its handler for processing
		 */
		skb = l3mdev_ip6_out((struct sock *)sk, skb);
		if (unlikely(!skb))
			return 0;

258 259 260
		/* hooks should never assume socket lock is held.
		 * we promote our socket to non const
		 */
261
		return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
262
			       net, (struct sock *)sk, skb, NULL, dst->dev,
263
			       dst_output);
L
Linus Torvalds 已提交
264 265 266
	}

	skb->dev = dst->dev;
267 268 269 270 271
	/* ipv6_local_error() does not require socket lock,
	 * we promote our socket to non const
	 */
	ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu);

E
Eric Dumazet 已提交
272
	IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
273 274 275
	kfree_skb(skb);
	return -EMSGSIZE;
}
276 277
EXPORT_SYMBOL(ip6_xmit);

L
Linus Torvalds 已提交
278 279 280 281 282 283 284 285
static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
{
	struct ip6_ra_chain *ra;
	struct sock *last = NULL;

	read_lock(&ip6_ra_lock);
	for (ra = ip6_ra_chain; ra; ra = ra->next) {
		struct sock *sk = ra->sk;
286 287 288
		if (sk && ra->sel == sel &&
		    (!sk->sk_bound_dev_if ||
		     sk->sk_bound_dev_if == skb->dev->ifindex)) {
L
Linus Torvalds 已提交
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
			if (last) {
				struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
				if (skb2)
					rawv6_rcv(last, skb2);
			}
			last = sk;
		}
	}

	if (last) {
		rawv6_rcv(last, skb);
		read_unlock(&ip6_ra_lock);
		return 1;
	}
	read_unlock(&ip6_ra_lock);
	return 0;
}

307 308
static int ip6_forward_proxy_check(struct sk_buff *skb)
{
309
	struct ipv6hdr *hdr = ipv6_hdr(skb);
310
	u8 nexthdr = hdr->nexthdr;
311
	__be16 frag_off;
312 313 314
	int offset;

	if (ipv6_ext_hdr(nexthdr)) {
315
		offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
316 317 318 319 320 321 322 323
		if (offset < 0)
			return 0;
	} else
		offset = sizeof(struct ipv6hdr);

	if (nexthdr == IPPROTO_ICMPV6) {
		struct icmp6hdr *icmp6;

324 325
		if (!pskb_may_pull(skb, (skb_network_header(skb) +
					 offset + 1 - skb->data)))
326 327
			return 0;

328
		icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345

		switch (icmp6->icmp6_type) {
		case NDISC_ROUTER_SOLICITATION:
		case NDISC_ROUTER_ADVERTISEMENT:
		case NDISC_NEIGHBOUR_SOLICITATION:
		case NDISC_NEIGHBOUR_ADVERTISEMENT:
		case NDISC_REDIRECT:
			/* For reaction involving unicast neighbor discovery
			 * message destined to the proxied address, pass it to
			 * input function.
			 */
			return 1;
		default:
			break;
		}
	}

346 347 348 349 350 351 352 353 354 355
	/*
	 * The proxying router can't forward traffic sent to a link-local
	 * address, so signal the sender and discard the packet. This
	 * behavior is clarified by the MIPv6 specification.
	 */
	if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
		dst_link_failure(skb);
		return -1;
	}

356 357 358
	return 0;
}

359 360
static inline int ip6_forward_finish(struct net *net, struct sock *sk,
				     struct sk_buff *skb)
L
Linus Torvalds 已提交
361
{
362
	return dst_output(net, sk, skb);
L
Linus Torvalds 已提交
363 364
}

365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
static unsigned int ip6_dst_mtu_forward(const struct dst_entry *dst)
{
	unsigned int mtu;
	struct inet6_dev *idev;

	if (dst_metric_locked(dst, RTAX_MTU)) {
		mtu = dst_metric_raw(dst, RTAX_MTU);
		if (mtu)
			return mtu;
	}

	mtu = IPV6_MIN_MTU;
	rcu_read_lock();
	idev = __in6_dev_get(dst->dev);
	if (idev)
		mtu = idev->cnf.mtu6;
	rcu_read_unlock();

	return mtu;
}

386 387
static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
{
388
	if (skb->len <= mtu)
389 390
		return false;

W
WANG Cong 已提交
391
	/* ipv6 conntrack defrag sets max_frag_size + ignore_df */
392 393 394
	if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)
		return true;

W
WANG Cong 已提交
395
	if (skb->ignore_df)
396 397
		return false;

398
	if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu))
399 400 401 402 403
		return false;

	return true;
}

L
Linus Torvalds 已提交
404 405
int ip6_forward(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
406
	struct dst_entry *dst = skb_dst(skb);
407
	struct ipv6hdr *hdr = ipv6_hdr(skb);
L
Linus Torvalds 已提交
408
	struct inet6_skb_parm *opt = IP6CB(skb);
409
	struct net *net = dev_net(dst->dev);
410
	u32 mtu;
411

412
	if (net->ipv6.devconf_all->forwarding == 0)
L
Linus Torvalds 已提交
413 414
		goto error;

415 416 417
	if (skb->pkt_type != PACKET_HOST)
		goto drop;

418 419 420
	if (unlikely(skb->sk))
		goto drop;

421 422 423
	if (skb_warn_if_lro(skb))
		goto drop;

L
Linus Torvalds 已提交
424
	if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
E
Eric Dumazet 已提交
425 426
		__IP6_INC_STATS(net, ip6_dst_idev(dst),
				IPSTATS_MIB_INDISCARDS);
L
Linus Torvalds 已提交
427 428 429
		goto drop;
	}

430
	skb_forward_csum(skb);
L
Linus Torvalds 已提交
431 432 433 434 435 436 437 438 439 440 441 442 443 444

	/*
	 *	We DO NOT make any processing on
	 *	RA packets, pushing them to user level AS IS
	 *	without ane WARRANTY that application will be able
	 *	to interpret them. The reason is that we
	 *	cannot make anything clever here.
	 *
	 *	We are not end-node, so that if packet contains
	 *	AH/ESP, we cannot make anything.
	 *	Defragmentation also would be mistake, RA packets
	 *	cannot be fragmented, because there is no warranty
	 *	that different fragments will go along one path. --ANK
	 */
445 446
	if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
		if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
L
Linus Torvalds 已提交
447 448 449 450 451 452 453 454 455
			return 0;
	}

	/*
	 *	check and decrement ttl
	 */
	if (hdr->hop_limit <= 1) {
		/* Force OUTPUT device used as source address */
		skb->dev = dst->dev;
456
		icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
E
Eric Dumazet 已提交
457 458
		__IP6_INC_STATS(net, ip6_dst_idev(dst),
				IPSTATS_MIB_INHDRERRORS);
L
Linus Torvalds 已提交
459 460 461 462 463

		kfree_skb(skb);
		return -ETIMEDOUT;
	}

464
	/* XXX: idev->cnf.proxy_ndp? */
465
	if (net->ipv6.devconf_all->proxy_ndp &&
466
	    pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) {
467 468
		int proxied = ip6_forward_proxy_check(skb);
		if (proxied > 0)
469
			return ip6_input(skb);
470
		else if (proxied < 0) {
E
Eric Dumazet 已提交
471 472
			__IP6_INC_STATS(net, ip6_dst_idev(dst),
					IPSTATS_MIB_INDISCARDS);
473 474
			goto drop;
		}
475 476
	}

L
Linus Torvalds 已提交
477
	if (!xfrm6_route_forward(skb)) {
E
Eric Dumazet 已提交
478 479
		__IP6_INC_STATS(net, ip6_dst_idev(dst),
				IPSTATS_MIB_INDISCARDS);
L
Linus Torvalds 已提交
480 481
		goto drop;
	}
E
Eric Dumazet 已提交
482
	dst = skb_dst(skb);
L
Linus Torvalds 已提交
483 484 485

	/* IPv6 specs say nothing about it, but it is clear that we cannot
	   send redirects to source routed frames.
486
	   We don't send redirects to frames decapsulated from IPsec.
L
Linus Torvalds 已提交
487
	 */
488
	if (skb->dev == dst->dev && opt->srcrt == 0 && !skb_sec_path(skb)) {
L
Linus Torvalds 已提交
489
		struct in6_addr *target = NULL;
490
		struct inet_peer *peer;
L
Linus Torvalds 已提交
491 492 493 494 495 496 497 498
		struct rt6_info *rt;

		/*
		 *	incoming and outgoing devices are the same
		 *	send a redirect.
		 */

		rt = (struct rt6_info *) dst;
499 500
		if (rt->rt6i_flags & RTF_GATEWAY)
			target = &rt->rt6i_gateway;
L
Linus Torvalds 已提交
501 502 503
		else
			target = &hdr->daddr;

504
		peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1);
505

L
Linus Torvalds 已提交
506 507 508
		/* Limit redirects both by destination (here)
		   and by source (inside ndisc_send_redirect)
		 */
509
		if (inet_peer_xrlim_allow(peer, 1*HZ))
510
			ndisc_send_redirect(skb, target);
511 512
		if (peer)
			inet_putpeer(peer);
513 514 515
	} else {
		int addrtype = ipv6_addr_type(&hdr->saddr);

L
Linus Torvalds 已提交
516
		/* This check is security critical. */
517 518
		if (addrtype == IPV6_ADDR_ANY ||
		    addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
519 520 521
			goto error;
		if (addrtype & IPV6_ADDR_LINKLOCAL) {
			icmpv6_send(skb, ICMPV6_DEST_UNREACH,
522
				    ICMPV6_NOT_NEIGHBOUR, 0);
523 524
			goto error;
		}
L
Linus Torvalds 已提交
525 526
	}

527
	mtu = ip6_dst_mtu_forward(dst);
528 529 530
	if (mtu < IPV6_MIN_MTU)
		mtu = IPV6_MIN_MTU;

531
	if (ip6_pkt_too_big(skb, mtu)) {
L
Linus Torvalds 已提交
532 533
		/* Again, force OUTPUT device used as source address */
		skb->dev = dst->dev;
534
		icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
E
Eric Dumazet 已提交
535 536 537 538
		__IP6_INC_STATS(net, ip6_dst_idev(dst),
				IPSTATS_MIB_INTOOBIGERRORS);
		__IP6_INC_STATS(net, ip6_dst_idev(dst),
				IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
539 540 541 542 543
		kfree_skb(skb);
		return -EMSGSIZE;
	}

	if (skb_cow(skb, dst->dev->hard_header_len)) {
E
Eric Dumazet 已提交
544 545
		__IP6_INC_STATS(net, ip6_dst_idev(dst),
				IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
546 547 548
		goto drop;
	}

549
	hdr = ipv6_hdr(skb);
L
Linus Torvalds 已提交
550 551

	/* Mangling hops number delayed to point after skb COW */
552

L
Linus Torvalds 已提交
553 554
	hdr->hop_limit--;

E
Eric Dumazet 已提交
555 556
	__IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
	__IP6_ADD_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len);
557 558
	return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD,
		       net, NULL, skb, skb->dev, dst->dev,
559
		       ip6_forward_finish);
L
Linus Torvalds 已提交
560 561

error:
E
Eric Dumazet 已提交
562
	__IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_INADDRERRORS);
L
Linus Torvalds 已提交
563 564 565 566 567 568 569 570 571 572
drop:
	kfree_skb(skb);
	return -EINVAL;
}

static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
{
	to->pkt_type = from->pkt_type;
	to->priority = from->priority;
	to->protocol = from->protocol;
E
Eric Dumazet 已提交
573 574
	skb_dst_drop(to);
	skb_dst_set(to, dst_clone(skb_dst(from)));
L
Linus Torvalds 已提交
575
	to->dev = from->dev;
T
Thomas Graf 已提交
576
	to->mark = from->mark;
L
Linus Torvalds 已提交
577 578 579 580

#ifdef CONFIG_NET_SCHED
	to->tc_index = from->tc_index;
#endif
581
	nf_copy(to, from);
582
	skb_copy_secmark(to, from);
L
Linus Torvalds 已提交
583 584
}

585 586
int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
		 int (*output)(struct net *, struct sock *, struct sk_buff *))
L
Linus Torvalds 已提交
587 588
{
	struct sk_buff *frag;
589
	struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
590 591
	struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
				inet6_sk(skb->sk) : NULL;
L
Linus Torvalds 已提交
592 593 594
	struct ipv6hdr *tmp_hdr;
	struct frag_hdr *fh;
	unsigned int mtu, hlen, left, len;
595
	int hroom, troom;
596
	__be32 frag_id;
597
	int ptr, offset = 0, err = 0;
L
Linus Torvalds 已提交
598 599 600 601 602
	u8 *prevhdr, nexthdr = 0;

	hlen = ip6_find_1stfragopt(skb, &prevhdr);
	nexthdr = *prevhdr;

J
John Heffner 已提交
603
	mtu = ip6_skb_dst_mtu(skb);
604 605

	/* We must not fragment if the socket is set to force MTU discovery
606
	 * or if the skb it not generated by a local socket.
607
	 */
608 609
	if (unlikely(!skb->ignore_df && skb->len > mtu))
		goto fail_toobig;
610

611 612 613 614 615 616 617 618
	if (IP6CB(skb)->frag_max_size) {
		if (IP6CB(skb)->frag_max_size > mtu)
			goto fail_toobig;

		/* don't send fragments larger than what we received */
		mtu = IP6CB(skb)->frag_max_size;
		if (mtu < IPV6_MIN_MTU)
			mtu = IPV6_MIN_MTU;
619 620
	}

621 622 623 624
	if (np && np->frag_size < mtu) {
		if (np->frag_size)
			mtu = np->frag_size;
	}
625
	if (mtu < hlen + sizeof(struct frag_hdr) + 8)
626
		goto fail_toobig;
627
	mtu -= hlen + sizeof(struct frag_hdr);
L
Linus Torvalds 已提交
628

629 630
	frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
				    &ipv6_hdr(skb)->saddr);
631

632 633 634 635
	if (skb->ip_summed == CHECKSUM_PARTIAL &&
	    (err = skb_checksum_help(skb)))
		goto fail;

636
	hroom = LL_RESERVED_SPACE(rt->dst.dev);
637
	if (skb_has_frag_list(skb)) {
638
		unsigned int first_len = skb_pagelen(skb);
639
		struct sk_buff *frag2;
L
Linus Torvalds 已提交
640 641 642

		if (first_len - hlen > mtu ||
		    ((first_len - hlen) & 7) ||
643 644
		    skb_cloned(skb) ||
		    skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
L
Linus Torvalds 已提交
645 646
			goto slow_path;

647
		skb_walk_frags(skb, frag) {
L
Linus Torvalds 已提交
648 649 650
			/* Correct geometry. */
			if (frag->len > mtu ||
			    ((frag->len & 7) && frag->next) ||
651
			    skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr)))
652
				goto slow_path_clean;
L
Linus Torvalds 已提交
653 654 655

			/* Partially cloned skb? */
			if (skb_shared(frag))
656
				goto slow_path_clean;
657 658 659 660 661 662

			BUG_ON(frag->sk);
			if (skb->sk) {
				frag->sk = skb->sk;
				frag->destructor = sock_wfree;
			}
663
			skb->truesize -= frag->truesize;
L
Linus Torvalds 已提交
664 665 666 667 668 669
		}

		err = 0;
		offset = 0;
		/* BUILD HEADER */

Y
YOSHIFUJI Hideaki 已提交
670
		*prevhdr = NEXTHDR_FRAGMENT;
671
		tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
L
Linus Torvalds 已提交
672
		if (!tmp_hdr) {
E
Eric Dumazet 已提交
673
			IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
674
				      IPSTATS_MIB_FRAGFAILS);
675 676
			err = -ENOMEM;
			goto fail;
L
Linus Torvalds 已提交
677
		}
678 679
		frag = skb_shinfo(skb)->frag_list;
		skb_frag_list_init(skb);
L
Linus Torvalds 已提交
680 681

		__skb_pull(skb, hlen);
682
		fh = (struct frag_hdr *)__skb_push(skb, sizeof(struct frag_hdr));
683 684
		__skb_push(skb, hlen);
		skb_reset_network_header(skb);
685
		memcpy(skb_network_header(skb), tmp_hdr, hlen);
L
Linus Torvalds 已提交
686 687 688 689

		fh->nexthdr = nexthdr;
		fh->reserved = 0;
		fh->frag_off = htons(IP6_MF);
690
		fh->identification = frag_id;
L
Linus Torvalds 已提交
691 692 693 694

		first_len = skb_pagelen(skb);
		skb->data_len = first_len - skb_headlen(skb);
		skb->len = first_len;
695 696
		ipv6_hdr(skb)->payload_len = htons(first_len -
						   sizeof(struct ipv6hdr));
697

698
		dst_hold(&rt->dst);
L
Linus Torvalds 已提交
699 700 701 702 703 704

		for (;;) {
			/* Prepare header of the next frame,
			 * before previous one went down. */
			if (frag) {
				frag->ip_summed = CHECKSUM_NONE;
705
				skb_reset_transport_header(frag);
706
				fh = (struct frag_hdr *)__skb_push(frag, sizeof(struct frag_hdr));
707 708
				__skb_push(frag, hlen);
				skb_reset_network_header(frag);
709 710
				memcpy(skb_network_header(frag), tmp_hdr,
				       hlen);
L
Linus Torvalds 已提交
711 712 713 714
				offset += skb->len - hlen - sizeof(struct frag_hdr);
				fh->nexthdr = nexthdr;
				fh->reserved = 0;
				fh->frag_off = htons(offset);
715
				if (frag->next)
L
Linus Torvalds 已提交
716 717
					fh->frag_off |= htons(IP6_MF);
				fh->identification = frag_id;
718 719 720
				ipv6_hdr(frag)->payload_len =
						htons(frag->len -
						      sizeof(struct ipv6hdr));
L
Linus Torvalds 已提交
721 722
				ip6_copy_metadata(frag, skb);
			}
723

724
			err = output(net, sk, skb);
725
			if (!err)
726
				IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
727
					      IPSTATS_MIB_FRAGCREATES);
728

L
Linus Torvalds 已提交
729 730 731 732 733 734 735 736
			if (err || !frag)
				break;

			skb = frag;
			frag = skb->next;
			skb->next = NULL;
		}

J
Jesper Juhl 已提交
737
		kfree(tmp_hdr);
L
Linus Torvalds 已提交
738 739

		if (err == 0) {
740
			IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
741
				      IPSTATS_MIB_FRAGOKS);
A
Amerigo Wang 已提交
742
			ip6_rt_put(rt);
L
Linus Torvalds 已提交
743 744 745
			return 0;
		}

746
		kfree_skb_list(frag);
L
Linus Torvalds 已提交
747

748
		IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
749
			      IPSTATS_MIB_FRAGFAILS);
A
Amerigo Wang 已提交
750
		ip6_rt_put(rt);
L
Linus Torvalds 已提交
751
		return err;
752 753 754 755 756 757 758 759 760

slow_path_clean:
		skb_walk_frags(skb, frag2) {
			if (frag2 == frag)
				break;
			frag2->sk = NULL;
			frag2->destructor = NULL;
			skb->truesize += frag2->truesize;
		}
L
Linus Torvalds 已提交
761 762 763 764 765 766 767 768 769 770 771
	}

slow_path:
	left = skb->len - hlen;		/* Space per frame */
	ptr = hlen;			/* Where to start from */

	/*
	 *	Fragment the datagram.
	 */

	*prevhdr = NEXTHDR_FRAGMENT;
772
	troom = rt->dst.dev->needed_tailroom;
L
Linus Torvalds 已提交
773 774 775 776

	/*
	 *	Keep copying data until we run out.
	 */
777
	while (left > 0)	{
L
Linus Torvalds 已提交
778 779 780 781
		len = left;
		/* IF: it doesn't fit, use 'mtu' - the data space left */
		if (len > mtu)
			len = mtu;
L
Lucas De Marchi 已提交
782
		/* IF: we are not sending up to and including the packet end
L
Linus Torvalds 已提交
783 784 785 786 787
		   then align the next start on an eight byte boundary */
		if (len < left)	{
			len &= ~7;
		}

788 789 790 791
		/* Allocate buffer */
		frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) +
				 hroom + troom, GFP_ATOMIC);
		if (!frag) {
E
Eric Dumazet 已提交
792
			IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
793
				      IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
794 795 796 797 798 799 800 801 802
			err = -ENOMEM;
			goto fail;
		}

		/*
		 *	Set up data on packet
		 */

		ip6_copy_metadata(frag, skb);
803
		skb_reserve(frag, hroom);
L
Linus Torvalds 已提交
804
		skb_put(frag, len + hlen + sizeof(struct frag_hdr));
805
		skb_reset_network_header(frag);
806
		fh = (struct frag_hdr *)(skb_network_header(frag) + hlen);
807 808
		frag->transport_header = (frag->network_header + hlen +
					  sizeof(struct frag_hdr));
L
Linus Torvalds 已提交
809 810 811 812 813 814 815 816 817 818 819

		/*
		 *	Charge the memory for the fragment to any owner
		 *	it might possess
		 */
		if (skb->sk)
			skb_set_owner_w(frag, skb->sk);

		/*
		 *	Copy the packet header into the new buffer.
		 */
820
		skb_copy_from_linear_data(skb, skb_network_header(frag), hlen);
L
Linus Torvalds 已提交
821 822 823 824 825 826

		/*
		 *	Build fragment header.
		 */
		fh->nexthdr = nexthdr;
		fh->reserved = 0;
827
		fh->identification = frag_id;
L
Linus Torvalds 已提交
828 829 830 831

		/*
		 *	Copy a block of the IP datagram.
		 */
H
Himangi Saraogi 已提交
832 833
		BUG_ON(skb_copy_bits(skb, ptr, skb_transport_header(frag),
				     len));
L
Linus Torvalds 已提交
834 835 836 837 838
		left -= len;

		fh->frag_off = htons(offset);
		if (left > 0)
			fh->frag_off |= htons(IP6_MF);
839 840
		ipv6_hdr(frag)->payload_len = htons(frag->len -
						    sizeof(struct ipv6hdr));
L
Linus Torvalds 已提交
841 842 843 844 845 846 847

		ptr += len;
		offset += len;

		/*
		 *	Put this fragment into the sending queue.
		 */
848
		err = output(net, sk, frag);
L
Linus Torvalds 已提交
849 850
		if (err)
			goto fail;
851

E
Eric Dumazet 已提交
852
		IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
853
			      IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
854
	}
E
Eric Dumazet 已提交
855
	IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
856
		      IPSTATS_MIB_FRAGOKS);
857
	consume_skb(skb);
L
Linus Torvalds 已提交
858 859
	return err;

860 861 862 863 864 865 866 867
fail_toobig:
	if (skb->sk && dst_allfrag(skb_dst(skb)))
		sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK);

	skb->dev = skb_dst(skb)->dev;
	icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
	err = -EMSGSIZE;

L
Linus Torvalds 已提交
868
fail:
E
Eric Dumazet 已提交
869
	IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
870
		      IPSTATS_MIB_FRAGFAILS);
871
	kfree_skb(skb);
L
Linus Torvalds 已提交
872 873 874
	return err;
}

875 876 877
static inline int ip6_rt_check(const struct rt6key *rt_key,
			       const struct in6_addr *fl_addr,
			       const struct in6_addr *addr_cache)
878
{
E
Eric Dumazet 已提交
879
	return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
880
		(!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache));
881 882
}

883 884
static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
					  struct dst_entry *dst,
885
					  const struct flowi6 *fl6)
L
Linus Torvalds 已提交
886
{
887
	struct ipv6_pinfo *np = inet6_sk(sk);
888
	struct rt6_info *rt;
L
Linus Torvalds 已提交
889

890 891 892
	if (!dst)
		goto out;

893 894 895 896 897 898
	if (dst->ops->family != AF_INET6) {
		dst_release(dst);
		return NULL;
	}

	rt = (struct rt6_info *)dst;
899 900 901
	/* Yes, checking route validity in not connected
	 * case is not very simple. Take into account,
	 * that we do not support routing by source, TOS,
902
	 * and MSG_DONTROUTE		--ANK (980726)
903
	 *
904 905
	 * 1. ip6_rt_check(): If route was host route,
	 *    check that cached destination is current.
906 907 908 909 910 911 912 913 914 915
	 *    If it is network route, we still may
	 *    check its validity using saved pointer
	 *    to the last used address: daddr_cache.
	 *    We do not want to save whole address now,
	 *    (because main consumer of this service
	 *    is tcp, which has not this problem),
	 *    so that the last trick works only on connected
	 *    sockets.
	 * 2. oif also should be the same.
	 */
916
	if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) ||
917
#ifdef CONFIG_IPV6_SUBTREES
918
	    ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) ||
919
#endif
D
David Ahern 已提交
920 921
	   (!(fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) &&
	      (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex))) {
922 923
		dst_release(dst);
		dst = NULL;
L
Linus Torvalds 已提交
924 925
	}

926 927 928 929
out:
	return dst;
}

930
static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
931
			       struct dst_entry **dst, struct flowi6 *fl6)
932
{
933 934
#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
	struct neighbour *n;
935
	struct rt6_info *rt;
936 937
#endif
	int err;
938
	int flags = 0;
939

940 941 942 943 944 945 946 947 948 949 950 951
	/* The correct way to handle this would be to do
	 * ip6_route_get_saddr, and then ip6_route_output; however,
	 * the route-specific preferred source forces the
	 * ip6_route_output call _before_ ip6_route_get_saddr.
	 *
	 * In source specific routing (no src=any default route),
	 * ip6_route_output will fail given src=any saddr, though, so
	 * that's why we try it again later.
	 */
	if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) {
		struct rt6_info *rt;
		bool had_dst = *dst != NULL;
L
Linus Torvalds 已提交
952

953 954 955
		if (!had_dst)
			*dst = ip6_route_output(net, sk, fl6);
		rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
956 957 958
		err = ip6_route_get_saddr(net, rt, &fl6->daddr,
					  sk ? inet6_sk(sk)->srcprefs : 0,
					  &fl6->saddr);
959
		if (err)
L
Linus Torvalds 已提交
960
			goto out_err_release;
961 962 963 964 965 966 967 968 969

		/* If we had an erroneous initial result, pretend it
		 * never existed and let the SA-enabled version take
		 * over.
		 */
		if (!had_dst && (*dst)->error) {
			dst_release(*dst);
			*dst = NULL;
		}
970 971 972

		if (fl6->flowi6_oif)
			flags |= RT6_LOOKUP_F_IFACE;
L
Linus Torvalds 已提交
973 974
	}

975
	if (!*dst)
976
		*dst = ip6_route_output_flags(net, sk, fl6, flags);
977 978 979 980 981

	err = (*dst)->error;
	if (err)
		goto out_err_release;

982
#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
983 984 985 986 987 988 989 990
	/*
	 * Here if the dst entry we've looked up
	 * has a neighbour entry that is in the INCOMPLETE
	 * state and the src address from the flow is
	 * marked as OPTIMISTIC, we release the found
	 * dst entry and replace it instead with the
	 * dst entry of the nexthop router
	 */
991
	rt = (struct rt6_info *) *dst;
992
	rcu_read_lock_bh();
993 994
	n = __ipv6_neigh_lookup_noref(rt->dst.dev,
				      rt6_nexthop(rt, &fl6->daddr));
995 996 997 998
	err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0;
	rcu_read_unlock_bh();

	if (err) {
999
		struct inet6_ifaddr *ifp;
1000
		struct flowi6 fl_gw6;
1001 1002
		int redirect;

1003
		ifp = ipv6_get_ifaddr(net, &fl6->saddr,
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
				      (*dst)->dev, 1);

		redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
		if (ifp)
			in6_ifa_put(ifp);

		if (redirect) {
			/*
			 * We need to get the dst entry for the
			 * default router instead
			 */
			dst_release(*dst);
1016 1017 1018
			memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
			memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
			*dst = ip6_route_output(net, sk, &fl_gw6);
1019 1020
			err = (*dst)->error;
			if (err)
1021
				goto out_err_release;
1022
		}
1023
	}
1024 1025
#endif

L
Linus Torvalds 已提交
1026 1027 1028 1029 1030
	return 0;

out_err_release:
	dst_release(*dst);
	*dst = NULL;
1031

1032 1033
	if (err == -ENETUNREACH)
		IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
L
Linus Torvalds 已提交
1034 1035
	return err;
}
A
Adrian Bunk 已提交
1036

1037 1038 1039 1040
/**
 *	ip6_dst_lookup - perform route lookup on flow
 *	@sk: socket which provides route info
 *	@dst: pointer to dst_entry * for result
1041
 *	@fl6: flow to lookup
1042 1043 1044 1045 1046
 *
 *	This function performs a route lookup on the given flow.
 *
 *	It returns zero on success, or a standard errno code on error.
 */
1047 1048
int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst,
		   struct flowi6 *fl6)
1049 1050
{
	*dst = NULL;
1051
	return ip6_dst_lookup_tail(net, sk, dst, fl6);
1052
}
1053 1054
EXPORT_SYMBOL_GPL(ip6_dst_lookup);

1055
/**
1056 1057
 *	ip6_dst_lookup_flow - perform route lookup on flow with ipsec
 *	@sk: socket which provides route info
1058
 *	@fl6: flow to lookup
1059 1060 1061 1062 1063 1064 1065
 *	@final_dst: final destination address for ipsec lookup
 *
 *	This function performs a route lookup on the given flow.
 *
 *	It returns a valid dst pointer on success, or a pointer encoded
 *	error code.
 */
1066
struct dst_entry *ip6_dst_lookup_flow(const struct sock *sk, struct flowi6 *fl6,
1067
				      const struct in6_addr *final_dst)
1068 1069 1070 1071
{
	struct dst_entry *dst = NULL;
	int err;

1072
	err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6);
1073 1074 1075
	if (err)
		return ERR_PTR(err);
	if (final_dst)
A
Alexey Dobriyan 已提交
1076
		fl6->daddr = *final_dst;
1077

1078
	return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0);
1079 1080 1081 1082 1083
}
EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);

/**
 *	ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1084
 *	@sk: socket which provides the dst cache and route info
1085
 *	@fl6: flow to lookup
1086
 *	@final_dst: final destination address for ipsec lookup
1087 1088 1089 1090 1091 1092
 *
 *	This function performs a route lookup on the given flow with the
 *	possibility of using the cached route in the socket if it is valid.
 *	It will take the socket dst lock when operating on the dst cache.
 *	As a result, this function can only be used in process context.
 *
1093 1094
 *	It returns a valid dst pointer on success, or a pointer encoded
 *	error code.
1095
 */
1096
struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1097
					 const struct in6_addr *final_dst)
1098
{
1099
	struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
1100

1101
	dst = ip6_sk_dst_check(sk, dst, fl6);
1102 1103
	if (!dst)
		dst = ip6_dst_lookup_flow(sk, fl6, final_dst);
1104

1105
	return dst;
1106
}
1107
EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow);
1108

A
Adrian Bunk 已提交
1109
static inline int ip6_ufo_append_data(struct sock *sk,
1110
			struct sk_buff_head *queue,
1111 1112 1113
			int getfrag(void *from, char *to, int offset, int len,
			int odd, struct sk_buff *skb),
			void *from, int length, int hh_len, int fragheaderlen,
1114 1115
			int exthdrlen, int transhdrlen, int mtu,
			unsigned int flags, const struct flowi6 *fl6)
1116 1117 1118 1119 1120 1121 1122 1123 1124

{
	struct sk_buff *skb;
	int err;

	/* There is support for UDP large send offload by network
	 * device, so create one single skb packet containing complete
	 * udp datagram
	 */
1125
	skb = skb_peek_tail(queue);
1126
	if (!skb) {
1127 1128 1129
		skb = sock_alloc_send_skb(sk,
			hh_len + fragheaderlen + transhdrlen + 20,
			(flags & MSG_DONTWAIT), &err);
1130
		if (!skb)
1131
			return err;
1132 1133 1134 1135 1136

		/* reserve space for Hardware header */
		skb_reserve(skb, hh_len);

		/* create space for UDP/IP header */
1137
		skb_put(skb, fragheaderlen + transhdrlen);
1138 1139

		/* initialize network header pointer */
1140
		skb_set_network_header(skb, exthdrlen);
1141 1142

		/* initialize protocol header pointer */
1143
		skb->transport_header = skb->network_header + fragheaderlen;
1144

1145
		skb->protocol = htons(ETH_P_IPV6);
1146 1147
		skb->csum = 0;

1148 1149 1150
		if (flags & MSG_CONFIRM)
			skb_set_dst_pending_confirm(skb, 1);

1151
		__skb_queue_tail(queue, skb);
1152 1153
	} else if (skb_is_gso(skb)) {
		goto append;
1154 1155
	}

1156 1157 1158 1159 1160 1161 1162
	skb->ip_summed = CHECKSUM_PARTIAL;
	/* Specify the length of each IPv6 datagram fragment.
	 * It has to be a multiple of 8.
	 */
	skb_shinfo(skb)->gso_size = (mtu - fragheaderlen -
				     sizeof(struct frag_hdr)) & ~7;
	skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1163 1164 1165
	skb_shinfo(skb)->ip6_frag_id = ipv6_select_ident(sock_net(sk),
							 &fl6->daddr,
							 &fl6->saddr);
1166 1167

append:
1168 1169
	return skb_append_datato_frags(sk, skb, getfrag, from,
				       (length - transhdrlen));
1170
}
L
Linus Torvalds 已提交
1171

1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
					       gfp_t gfp)
{
	return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
}

static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
						gfp_t gfp)
{
	return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
}

1184
static void ip6_append_data_mtu(unsigned int *mtu,
G
Gao feng 已提交
1185 1186 1187
				int *maxfraglen,
				unsigned int fragheaderlen,
				struct sk_buff *skb,
1188
				struct rt6_info *rt,
1189
				unsigned int orig_mtu)
G
Gao feng 已提交
1190 1191
{
	if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
1192
		if (!skb) {
G
Gao feng 已提交
1193
			/* first fragment, reserve header_len */
1194
			*mtu = orig_mtu - rt->dst.header_len;
G
Gao feng 已提交
1195 1196 1197 1198 1199 1200

		} else {
			/*
			 * this fragment is not first, the headers
			 * space is regarded as data space.
			 */
1201
			*mtu = orig_mtu;
G
Gao feng 已提交
1202 1203 1204 1205 1206 1207
		}
		*maxfraglen = ((*mtu - fragheaderlen) & ~7)
			      + fragheaderlen - sizeof(struct frag_hdr);
	}
}

1208
static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
W
Wei Wang 已提交
1209
			  struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6,
1210 1211 1212 1213
			  struct rt6_info *rt, struct flowi6 *fl6)
{
	struct ipv6_pinfo *np = inet6_sk(sk);
	unsigned int mtu;
W
Wei Wang 已提交
1214
	struct ipv6_txoptions *opt = ipc6->opt;
1215 1216 1217 1218 1219 1220 1221 1222 1223

	/*
	 * setup for corking
	 */
	if (opt) {
		if (WARN_ON(v6_cork->opt))
			return -EINVAL;

		v6_cork->opt = kzalloc(opt->tot_len, sk->sk_allocation);
1224
		if (unlikely(!v6_cork->opt))
1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
			return -ENOBUFS;

		v6_cork->opt->tot_len = opt->tot_len;
		v6_cork->opt->opt_flen = opt->opt_flen;
		v6_cork->opt->opt_nflen = opt->opt_nflen;

		v6_cork->opt->dst0opt = ip6_opt_dup(opt->dst0opt,
						    sk->sk_allocation);
		if (opt->dst0opt && !v6_cork->opt->dst0opt)
			return -ENOBUFS;

		v6_cork->opt->dst1opt = ip6_opt_dup(opt->dst1opt,
						    sk->sk_allocation);
		if (opt->dst1opt && !v6_cork->opt->dst1opt)
			return -ENOBUFS;

		v6_cork->opt->hopopt = ip6_opt_dup(opt->hopopt,
						   sk->sk_allocation);
		if (opt->hopopt && !v6_cork->opt->hopopt)
			return -ENOBUFS;

		v6_cork->opt->srcrt = ip6_rthdr_dup(opt->srcrt,
						    sk->sk_allocation);
		if (opt->srcrt && !v6_cork->opt->srcrt)
			return -ENOBUFS;

		/* need source address above miyazawa*/
	}
	dst_hold(&rt->dst);
	cork->base.dst = &rt->dst;
	cork->fl.u.ip6 = *fl6;
W
Wei Wang 已提交
1256 1257
	v6_cork->hop_limit = ipc6->hlimit;
	v6_cork->tclass = ipc6->tclass;
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
	if (rt->dst.flags & DST_XFRM_TUNNEL)
		mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
		      rt->dst.dev->mtu : dst_mtu(&rt->dst);
	else
		mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
		      rt->dst.dev->mtu : dst_mtu(rt->dst.path);
	if (np->frag_size < mtu) {
		if (np->frag_size)
			mtu = np->frag_size;
	}
	cork->base.fragsize = mtu;
	if (dst_allfrag(rt->dst.path))
		cork->base.flags |= IPCORK_ALLFRAG;
	cork->base.length = 0;

	return 0;
}

1276 1277 1278 1279 1280 1281 1282 1283 1284
static int __ip6_append_data(struct sock *sk,
			     struct flowi6 *fl6,
			     struct sk_buff_head *queue,
			     struct inet_cork *cork,
			     struct inet6_cork *v6_cork,
			     struct page_frag *pfrag,
			     int getfrag(void *from, char *to, int offset,
					 int len, int odd, struct sk_buff *skb),
			     void *from, int length, int transhdrlen,
W
Wei Wang 已提交
1285
			     unsigned int flags, struct ipcm6_cookie *ipc6,
1286
			     const struct sockcm_cookie *sockc)
L
Linus Torvalds 已提交
1287
{
G
Gao feng 已提交
1288
	struct sk_buff *skb, *skb_prev = NULL;
1289
	unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu;
1290 1291
	int exthdrlen = 0;
	int dst_exthdrlen = 0;
L
Linus Torvalds 已提交
1292 1293 1294 1295
	int hh_len;
	int copy;
	int err;
	int offset = 0;
1296
	__u8 tx_flags = 0;
1297
	u32 tskey = 0;
1298 1299
	struct rt6_info *rt = (struct rt6_info *)cork->dst;
	struct ipv6_txoptions *opt = v6_cork->opt;
1300
	int csummode = CHECKSUM_NONE;
1301
	unsigned int maxnonfragsize, headersize;
L
Linus Torvalds 已提交
1302

1303 1304 1305
	skb = skb_peek_tail(queue);
	if (!skb) {
		exthdrlen = opt ? opt->opt_flen : 0;
1306
		dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
L
Linus Torvalds 已提交
1307
	}
1308

1309
	mtu = cork->fragsize;
1310
	orig_mtu = mtu;
L
Linus Torvalds 已提交
1311

1312
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
L
Linus Torvalds 已提交
1313

1314
	fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
1315
			(opt ? opt->opt_nflen : 0);
1316 1317
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
		     sizeof(struct frag_hdr);
L
Linus Torvalds 已提交
1318

1319 1320 1321 1322 1323 1324
	headersize = sizeof(struct ipv6hdr) +
		     (opt ? opt->opt_flen + opt->opt_nflen : 0) +
		     (dst_allfrag(&rt->dst) ?
		      sizeof(struct frag_hdr) : 0) +
		     rt->rt6i_nfheader_len;

W
Wei Wang 已提交
1325
	if (cork->length + length > mtu - headersize && ipc6->dontfrag &&
1326 1327 1328 1329 1330 1331
	    (sk->sk_protocol == IPPROTO_UDP ||
	     sk->sk_protocol == IPPROTO_RAW)) {
		ipv6_local_rxpmtu(sk, fl6, mtu - headersize +
				sizeof(struct ipv6hdr));
		goto emsgsize;
	}
1332

1333 1334 1335 1336
	if (ip6_sk_ignore_df(sk))
		maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN;
	else
		maxnonfragsize = mtu;
1337

1338
	if (cork->length + length > maxnonfragsize - headersize) {
1339
emsgsize:
1340 1341 1342 1343
		ipv6_local_error(sk, EMSGSIZE, fl6,
				 mtu - headersize +
				 sizeof(struct ipv6hdr));
		return -EMSGSIZE;
L
Linus Torvalds 已提交
1344 1345
	}

1346 1347 1348 1349 1350
	/* CHECKSUM_PARTIAL only with no extension headers and when
	 * we are not going to fragment
	 */
	if (transhdrlen && sk->sk_protocol == IPPROTO_UDP &&
	    headersize == sizeof(struct ipv6hdr) &&
1351
	    length <= mtu - headersize &&
1352
	    !(flags & MSG_MORE) &&
1353
	    rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
1354 1355
		csummode = CHECKSUM_PARTIAL;

1356
	if (sk->sk_type == SOCK_DGRAM || sk->sk_type == SOCK_RAW) {
1357
		sock_tx_timestamp(sk, sockc->tsflags, &tx_flags);
1358 1359 1360 1361
		if (tx_flags & SKBTX_ANY_SW_TSTAMP &&
		    sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
			tskey = sk->sk_tskey++;
	}
1362

L
Linus Torvalds 已提交
1363 1364 1365 1366 1367 1368 1369
	/*
	 * Let's try using as much space as possible.
	 * Use MTU if total length of the message fits into the MTU.
	 * Otherwise, we need to reserve fragment header and
	 * fragment alignment (= 8-15 octects, in total).
	 *
	 * Note that we may need to "move" the data from the tail of
1370
	 * of the buffer to the new fragment when we split
L
Linus Torvalds 已提交
1371 1372
	 * the message.
	 *
1373
	 * FIXME: It may be fragmented into multiple chunks
L
Linus Torvalds 已提交
1374 1375
	 *        at once if non-fragmentable extension headers
	 *        are too large.
1376
	 * --yoshfuji
L
Linus Torvalds 已提交
1377 1378
	 */

1379
	cork->length += length;
1380
	if ((((length + fragheaderlen) > mtu) ||
1381 1382
	     (skb && skb_is_gso(skb))) &&
	    (sk->sk_protocol == IPPROTO_UDP) &&
1383
	    (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len &&
1384
	    (sk->sk_type == SOCK_DGRAM) && !udp_get_no_check6_tx(sk)) {
1385
		err = ip6_ufo_append_data(sk, queue, getfrag, from, length,
1386
					  hh_len, fragheaderlen, exthdrlen,
1387
					  transhdrlen, mtu, flags, fl6);
1388 1389 1390
		if (err)
			goto error;
		return 0;
1391
	}
L
Linus Torvalds 已提交
1392

1393
	if (!skb)
L
Linus Torvalds 已提交
1394 1395 1396 1397
		goto alloc_new_skb;

	while (length > 0) {
		/* Check if the remaining data fits into current packet. */
1398
		copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
L
Linus Torvalds 已提交
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
		if (copy < length)
			copy = maxfraglen - skb->len;

		if (copy <= 0) {
			char *data;
			unsigned int datalen;
			unsigned int fraglen;
			unsigned int fraggap;
			unsigned int alloclen;
alloc_new_skb:
			/* There's no room in the current skb */
G
Gao feng 已提交
1410 1411
			if (skb)
				fraggap = skb->len - maxfraglen;
L
Linus Torvalds 已提交
1412 1413
			else
				fraggap = 0;
G
Gao feng 已提交
1414
			/* update mtu and maxfraglen if necessary */
1415
			if (!skb || !skb_prev)
G
Gao feng 已提交
1416
				ip6_append_data_mtu(&mtu, &maxfraglen,
1417
						    fragheaderlen, skb, rt,
1418
						    orig_mtu);
G
Gao feng 已提交
1419 1420

			skb_prev = skb;
L
Linus Torvalds 已提交
1421 1422 1423 1424 1425 1426 1427

			/*
			 * If remaining data exceeds the mtu,
			 * we know we need more fragment(s).
			 */
			datalen = length + fraggap;

G
Gao feng 已提交
1428 1429
			if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
				datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
L
Linus Torvalds 已提交
1430
			if ((flags & MSG_MORE) &&
1431
			    !(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
1432 1433 1434 1435
				alloclen = mtu;
			else
				alloclen = datalen + fragheaderlen;

1436 1437
			alloclen += dst_exthdrlen;

G
Gao feng 已提交
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
			if (datalen != length + fraggap) {
				/*
				 * this is not the last fragment, the trailer
				 * space is regarded as data space.
				 */
				datalen += rt->dst.trailer_len;
			}

			alloclen += rt->dst.trailer_len;
			fraglen = datalen + fragheaderlen;
L
Linus Torvalds 已提交
1448 1449 1450

			/*
			 * We just reserve space for fragment header.
1451
			 * Note: this may be overallocation if the message
L
Linus Torvalds 已提交
1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
			 * (without MSG_MORE) fits into the MTU.
			 */
			alloclen += sizeof(struct frag_hdr);

			if (transhdrlen) {
				skb = sock_alloc_send_skb(sk,
						alloclen + hh_len,
						(flags & MSG_DONTWAIT), &err);
			} else {
				skb = NULL;
				if (atomic_read(&sk->sk_wmem_alloc) <=
				    2 * sk->sk_sndbuf)
					skb = sock_wmalloc(sk,
							   alloclen + hh_len, 1,
							   sk->sk_allocation);
1467
				if (unlikely(!skb))
L
Linus Torvalds 已提交
1468 1469
					err = -ENOBUFS;
			}
1470
			if (!skb)
L
Linus Torvalds 已提交
1471 1472 1473 1474
				goto error;
			/*
			 *	Fill in the control structures
			 */
1475
			skb->protocol = htons(ETH_P_IPV6);
1476
			skb->ip_summed = csummode;
L
Linus Torvalds 已提交
1477
			skb->csum = 0;
1478 1479 1480
			/* reserve for fragmentation and ipsec header */
			skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
				    dst_exthdrlen);
L
Linus Torvalds 已提交
1481

1482 1483 1484
			/* Only the initial fragment is time stamped */
			skb_shinfo(skb)->tx_flags = tx_flags;
			tx_flags = 0;
1485 1486
			skb_shinfo(skb)->tskey = tskey;
			tskey = 0;
1487

L
Linus Torvalds 已提交
1488 1489 1490
			/*
			 *	Find where to start putting bytes
			 */
1491 1492 1493
			data = skb_put(skb, fraglen);
			skb_set_network_header(skb, exthdrlen);
			data += fragheaderlen;
1494 1495
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
L
Linus Torvalds 已提交
1496 1497 1498 1499 1500 1501 1502
			if (fraggap) {
				skb->csum = skb_copy_and_csum_bits(
					skb_prev, maxfraglen,
					data + transhdrlen, fraggap, 0);
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
				data += fraggap;
1503
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1504 1505
			}
			copy = datalen - transhdrlen - fraggap;
1506

L
Linus Torvalds 已提交
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
			if (copy < 0) {
				err = -EINVAL;
				kfree_skb(skb);
				goto error;
			} else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
				err = -EFAULT;
				kfree_skb(skb);
				goto error;
			}

			offset += copy;
			length -= datalen - fraggap;
			transhdrlen = 0;
			exthdrlen = 0;
1521
			dst_exthdrlen = 0;
L
Linus Torvalds 已提交
1522

1523 1524 1525
			if ((flags & MSG_CONFIRM) && !skb_prev)
				skb_set_dst_pending_confirm(skb, 1);

L
Linus Torvalds 已提交
1526 1527 1528
			/*
			 * Put the packet on the pending queue
			 */
1529
			__skb_queue_tail(queue, skb);
L
Linus Torvalds 已提交
1530 1531 1532 1533 1534 1535
			continue;
		}

		if (copy > length)
			copy = length;

1536
		if (!(rt->dst.dev->features&NETIF_F_SG)) {
L
Linus Torvalds 已提交
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
			unsigned int off;

			off = skb->len;
			if (getfrag(from, skb_put(skb, copy),
						offset, copy, off, skb) < 0) {
				__skb_trim(skb, off);
				err = -EFAULT;
				goto error;
			}
		} else {
			int i = skb_shinfo(skb)->nr_frags;

1549 1550
			err = -ENOMEM;
			if (!sk_page_frag_refill(sk, pfrag))
L
Linus Torvalds 已提交
1551
				goto error;
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562

			if (!skb_can_coalesce(skb, i, pfrag->page,
					      pfrag->offset)) {
				err = -EMSGSIZE;
				if (i == MAX_SKB_FRAGS)
					goto error;

				__skb_fill_page_desc(skb, i, pfrag->page,
						     pfrag->offset, 0);
				skb_shinfo(skb)->nr_frags = ++i;
				get_page(pfrag->page);
L
Linus Torvalds 已提交
1563
			}
1564
			copy = min_t(int, copy, pfrag->size - pfrag->offset);
E
Eric Dumazet 已提交
1565
			if (getfrag(from,
1566 1567 1568 1569 1570 1571
				    page_address(pfrag->page) + pfrag->offset,
				    offset, copy, skb->len, skb) < 0)
				goto error_efault;

			pfrag->offset += copy;
			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
L
Linus Torvalds 已提交
1572 1573
			skb->len += copy;
			skb->data_len += copy;
1574 1575
			skb->truesize += copy;
			atomic_add(copy, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1576 1577 1578 1579
		}
		offset += copy;
		length -= copy;
	}
1580

L
Linus Torvalds 已提交
1581
	return 0;
1582 1583 1584

error_efault:
	err = -EFAULT;
L
Linus Torvalds 已提交
1585
error:
1586
	cork->length -= length;
1587
	IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1588 1589
	return err;
}
1590 1591 1592 1593

int ip6_append_data(struct sock *sk,
		    int getfrag(void *from, char *to, int offset, int len,
				int odd, struct sk_buff *skb),
W
Wei Wang 已提交
1594 1595 1596
		    void *from, int length, int transhdrlen,
		    struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
		    struct rt6_info *rt, unsigned int flags,
1597
		    const struct sockcm_cookie *sockc)
1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
{
	struct inet_sock *inet = inet_sk(sk);
	struct ipv6_pinfo *np = inet6_sk(sk);
	int exthdrlen;
	int err;

	if (flags&MSG_PROBE)
		return 0;
	if (skb_queue_empty(&sk->sk_write_queue)) {
		/*
		 * setup for corking
		 */
W
Wei Wang 已提交
1610 1611
		err = ip6_setup_cork(sk, &inet->cork, &np->cork,
				     ipc6, rt, fl6);
1612 1613 1614
		if (err)
			return err;

W
Wei Wang 已提交
1615
		exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1616 1617 1618 1619 1620 1621 1622 1623 1624
		length += exthdrlen;
		transhdrlen += exthdrlen;
	} else {
		fl6 = &inet->cork.fl.u.ip6;
		transhdrlen = 0;
	}

	return __ip6_append_data(sk, fl6, &sk->sk_write_queue, &inet->cork.base,
				 &np->cork, sk_page_frag(sk), getfrag,
W
Wei Wang 已提交
1625
				 from, length, transhdrlen, flags, ipc6, sockc);
1626
}
1627
EXPORT_SYMBOL_GPL(ip6_append_data);
L
Linus Torvalds 已提交
1628

1629 1630
static void ip6_cork_release(struct inet_cork_full *cork,
			     struct inet6_cork *v6_cork)
1631
{
1632 1633 1634 1635 1636 1637 1638
	if (v6_cork->opt) {
		kfree(v6_cork->opt->dst0opt);
		kfree(v6_cork->opt->dst1opt);
		kfree(v6_cork->opt->hopopt);
		kfree(v6_cork->opt->srcrt);
		kfree(v6_cork->opt);
		v6_cork->opt = NULL;
1639 1640
	}

1641 1642 1643 1644
	if (cork->base.dst) {
		dst_release(cork->base.dst);
		cork->base.dst = NULL;
		cork->base.flags &= ~IPCORK_ALLFRAG;
1645
	}
1646
	memset(&cork->fl, 0, sizeof(cork->fl));
1647 1648
}

V
Vlad Yasevich 已提交
1649 1650 1651 1652
struct sk_buff *__ip6_make_skb(struct sock *sk,
			       struct sk_buff_head *queue,
			       struct inet_cork_full *cork,
			       struct inet6_cork *v6_cork)
L
Linus Torvalds 已提交
1653 1654 1655 1656 1657
{
	struct sk_buff *skb, *tmp_skb;
	struct sk_buff **tail_skb;
	struct in6_addr final_dst_buf, *final_dst = &final_dst_buf;
	struct ipv6_pinfo *np = inet6_sk(sk);
1658
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
1659
	struct ipv6hdr *hdr;
V
Vlad Yasevich 已提交
1660 1661 1662
	struct ipv6_txoptions *opt = v6_cork->opt;
	struct rt6_info *rt = (struct rt6_info *)cork->base.dst;
	struct flowi6 *fl6 = &cork->fl.u.ip6;
1663
	unsigned char proto = fl6->flowi6_proto;
L
Linus Torvalds 已提交
1664

V
Vlad Yasevich 已提交
1665
	skb = __skb_dequeue(queue);
1666
	if (!skb)
L
Linus Torvalds 已提交
1667 1668 1669 1670
		goto out;
	tail_skb = &(skb_shinfo(skb)->frag_list);

	/* move skb->data to ip header from ext header */
1671
	if (skb->data < skb_network_header(skb))
1672
		__skb_pull(skb, skb_network_offset(skb));
V
Vlad Yasevich 已提交
1673
	while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1674
		__skb_pull(tmp_skb, skb_network_header_len(skb));
L
Linus Torvalds 已提交
1675 1676 1677 1678 1679 1680 1681 1682 1683
		*tail_skb = tmp_skb;
		tail_skb = &(tmp_skb->next);
		skb->len += tmp_skb->len;
		skb->data_len += tmp_skb->len;
		skb->truesize += tmp_skb->truesize;
		tmp_skb->destructor = NULL;
		tmp_skb->sk = NULL;
	}

1684
	/* Allow local fragmentation. */
W
WANG Cong 已提交
1685
	skb->ignore_df = ip6_sk_ignore_df(sk);
1686

A
Alexey Dobriyan 已提交
1687
	*final_dst = fl6->daddr;
1688
	__skb_pull(skb, skb_network_header_len(skb));
L
Linus Torvalds 已提交
1689 1690 1691
	if (opt && opt->opt_flen)
		ipv6_push_frag_opts(skb, opt, &proto);
	if (opt && opt->opt_nflen)
1692
		ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr);
L
Linus Torvalds 已提交
1693

1694 1695
	skb_push(skb, sizeof(struct ipv6hdr));
	skb_reset_network_header(skb);
1696
	hdr = ipv6_hdr(skb);
1697

V
Vlad Yasevich 已提交
1698
	ip6_flow_hdr(hdr, v6_cork->tclass,
1699
		     ip6_make_flowlabel(net, skb, fl6->flowlabel,
1700
					np->autoflowlabel, fl6));
V
Vlad Yasevich 已提交
1701
	hdr->hop_limit = v6_cork->hop_limit;
L
Linus Torvalds 已提交
1702
	hdr->nexthdr = proto;
A
Alexey Dobriyan 已提交
1703 1704
	hdr->saddr = fl6->saddr;
	hdr->daddr = *final_dst;
L
Linus Torvalds 已提交
1705

1706
	skb->priority = sk->sk_priority;
1707
	skb->mark = sk->sk_mark;
1708

1709
	skb_dst_set(skb, dst_clone(&rt->dst));
1710
	IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len);
1711
	if (proto == IPPROTO_ICMPV6) {
E
Eric Dumazet 已提交
1712
		struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
1713

1714 1715
		ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type);
		ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
1716 1717
	}

V
Vlad Yasevich 已提交
1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728
	ip6_cork_release(cork, v6_cork);
out:
	return skb;
}

int ip6_send_skb(struct sk_buff *skb)
{
	struct net *net = sock_net(skb->sk);
	struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
	int err;

1729
	err = ip6_local_out(net, skb->sk, skb);
L
Linus Torvalds 已提交
1730 1731
	if (err) {
		if (err > 0)
E
Eric Dumazet 已提交
1732
			err = net_xmit_errno(err);
L
Linus Torvalds 已提交
1733
		if (err)
V
Vlad Yasevich 已提交
1734 1735
			IP6_INC_STATS(net, rt->rt6i_idev,
				      IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1736 1737 1738
	}

	return err;
V
Vlad Yasevich 已提交
1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
}

int ip6_push_pending_frames(struct sock *sk)
{
	struct sk_buff *skb;

	skb = ip6_finish_skb(sk);
	if (!skb)
		return 0;

	return ip6_send_skb(skb);
L
Linus Torvalds 已提交
1750
}
1751
EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
L
Linus Torvalds 已提交
1752

1753
static void __ip6_flush_pending_frames(struct sock *sk,
V
Vlad Yasevich 已提交
1754 1755 1756
				       struct sk_buff_head *queue,
				       struct inet_cork_full *cork,
				       struct inet6_cork *v6_cork)
L
Linus Torvalds 已提交
1757 1758 1759
{
	struct sk_buff *skb;

1760
	while ((skb = __skb_dequeue_tail(queue)) != NULL) {
E
Eric Dumazet 已提交
1761 1762
		if (skb_dst(skb))
			IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
1763
				      IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1764 1765 1766
		kfree_skb(skb);
	}

V
Vlad Yasevich 已提交
1767
	ip6_cork_release(cork, v6_cork);
L
Linus Torvalds 已提交
1768
}
1769 1770 1771

void ip6_flush_pending_frames(struct sock *sk)
{
V
Vlad Yasevich 已提交
1772 1773
	__ip6_flush_pending_frames(sk, &sk->sk_write_queue,
				   &inet_sk(sk)->cork, &inet6_sk(sk)->cork);
1774
}
1775
EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
V
Vlad Yasevich 已提交
1776 1777 1778 1779 1780

struct sk_buff *ip6_make_skb(struct sock *sk,
			     int getfrag(void *from, char *to, int offset,
					 int len, int odd, struct sk_buff *skb),
			     void *from, int length, int transhdrlen,
W
Wei Wang 已提交
1781
			     struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
V
Vlad Yasevich 已提交
1782
			     struct rt6_info *rt, unsigned int flags,
W
Wei Wang 已提交
1783
			     const struct sockcm_cookie *sockc)
V
Vlad Yasevich 已提交
1784 1785 1786 1787
{
	struct inet_cork_full cork;
	struct inet6_cork v6_cork;
	struct sk_buff_head queue;
W
Wei Wang 已提交
1788
	int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
V
Vlad Yasevich 已提交
1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	int err;

	if (flags & MSG_PROBE)
		return NULL;

	__skb_queue_head_init(&queue);

	cork.base.flags = 0;
	cork.base.addr = 0;
	cork.base.opt = NULL;
	v6_cork.opt = NULL;
W
Wei Wang 已提交
1800
	err = ip6_setup_cork(sk, &cork, &v6_cork, ipc6, rt, fl6);
V
Vlad Yasevich 已提交
1801 1802 1803
	if (err)
		return ERR_PTR(err);

W
Wei Wang 已提交
1804 1805
	if (ipc6->dontfrag < 0)
		ipc6->dontfrag = inet6_sk(sk)->dontfrag;
V
Vlad Yasevich 已提交
1806 1807 1808 1809

	err = __ip6_append_data(sk, fl6, &queue, &cork.base, &v6_cork,
				&current->task_frag, getfrag, from,
				length + exthdrlen, transhdrlen + exthdrlen,
W
Wei Wang 已提交
1810
				flags, ipc6, sockc);
V
Vlad Yasevich 已提交
1811 1812 1813 1814 1815 1816 1817
	if (err) {
		__ip6_flush_pending_frames(sk, &queue, &cork, &v6_cork);
		return ERR_PTR(err);
	}

	return __ip6_make_skb(sk, &queue, &cork, &v6_cork);
}