ip6_output.c 44.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 *	IPv6 output functions
3
 *	Linux INET6 implementation
L
Linus Torvalds 已提交
4 5
 *
 *	Authors:
6
 *	Pedro Roque		<roque@di.fc.ul.pt>
L
Linus Torvalds 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *
 *	Based on linux/net/ipv4/ip_output.c
 *
 *	This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 *
 *	Changes:
 *	A.N.Kuznetsov	:	airthmetics in fragmentation.
 *				extension headers are implemented.
 *				route changes now work.
 *				ip6_forward does not confuse sniffers.
 *				etc.
 *
 *      H. von Brand    :       Added missing #include <linux/string.h>
23
 *	Imran Patel	:	frag id should be in NBO
L
Linus Torvalds 已提交
24 25 26 27 28 29
 *      Kazunori MIYAZAWA @USAGI
 *			:       add ip6_append_data and related functions
 *				for datagram xmit
 */

#include <linux/errno.h>
H
Herbert Xu 已提交
30
#include <linux/kernel.h>
L
Linus Torvalds 已提交
31 32 33 34 35 36 37 38
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/in6.h>
#include <linux/tcp.h>
#include <linux/route.h>
39
#include <linux/module.h>
40
#include <linux/slab.h>
L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

#include <linux/netfilter.h>
#include <linux/netfilter_ipv6.h>

#include <net/sock.h>
#include <net/snmp.h>

#include <net/ipv6.h>
#include <net/ndisc.h>
#include <net/protocol.h>
#include <net/ip6_route.h>
#include <net/addrconf.h>
#include <net/rawv6.h>
#include <net/icmp.h>
#include <net/xfrm.h>
#include <net/checksum.h>
57
#include <linux/mroute6.h>
L
Linus Torvalds 已提交
58

59
static int ip6_finish_output2(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
60
{
E
Eric Dumazet 已提交
61
	struct dst_entry *dst = skb_dst(skb);
L
Linus Torvalds 已提交
62
	struct net_device *dev = dst->dev;
63
	struct neighbour *neigh;
64 65
	struct in6_addr *nexthop;
	int ret;
L
Linus Torvalds 已提交
66 67 68 69

	skb->protocol = htons(ETH_P_IPV6);
	skb->dev = dev;

70
	if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
E
Eric Dumazet 已提交
71
		struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
L
Linus Torvalds 已提交
72

73
		if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) &&
74
		    ((mroute6_socket(dev_net(dev), skb) &&
75
		     !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
76 77
		     ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr,
					 &ipv6_hdr(skb)->saddr))) {
L
Linus Torvalds 已提交
78 79 80 81 82 83
			struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);

			/* Do not check for IFF_ALLMULTI; multicast routing
			   is not supported in any case.
			 */
			if (newskb)
84
				NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
85
					sk, newskb, NULL, newskb->dev,
86
					dev_loopback_xmit);
L
Linus Torvalds 已提交
87

88
			if (ipv6_hdr(skb)->hop_limit == 0) {
89 90
				IP6_INC_STATS(dev_net(dev), idev,
					      IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
91 92 93 94 95
				kfree_skb(skb);
				return 0;
			}
		}

96 97
		IP6_UPD_PO_STATS(dev_net(dev), idev, IPSTATS_MIB_OUTMCAST,
				skb->len);
98 99 100 101 102 103 104

		if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <=
		    IPV6_ADDR_SCOPE_NODELOCAL &&
		    !(dev->flags & IFF_LOOPBACK)) {
			kfree_skb(skb);
			return 0;
		}
L
Linus Torvalds 已提交
105 106
	}

107
	rcu_read_lock_bh();
108
	nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr);
109 110 111 112 113 114 115 116 117
	neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
	if (unlikely(!neigh))
		neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
	if (!IS_ERR(neigh)) {
		ret = dst_neigh_output(dst, neigh, skb);
		rcu_read_unlock_bh();
		return ret;
	}
	rcu_read_unlock_bh();
118

119 120
	IP6_INC_STATS(dev_net(dst->dev),
		      ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
121 122
	kfree_skb(skb);
	return -EINVAL;
L
Linus Torvalds 已提交
123 124
}

125
static int ip6_finish_output(struct sock *sk, struct sk_buff *skb)
126 127
{
	if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) ||
128 129
	    dst_allfrag(skb_dst(skb)) ||
	    (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))
130
		return ip6_fragment(sk, skb, ip6_finish_output2);
131
	else
132
		return ip6_finish_output2(sk, skb);
133 134
}

135
int ip6_output(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
136
{
137
	struct net_device *dev = skb_dst(skb)->dev;
E
Eric Dumazet 已提交
138
	struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
139
	if (unlikely(idev->cnf.disable_ipv6)) {
140
		IP6_INC_STATS(dev_net(dev), idev,
141
			      IPSTATS_MIB_OUTDISCARDS);
142 143 144 145
		kfree_skb(skb);
		return 0;
	}

146 147
	return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, sk, skb,
			    NULL, dev,
148 149
			    ip6_finish_output,
			    !(IP6CB(skb)->flags & IP6SKB_REROUTED));
L
Linus Torvalds 已提交
150 151 152
}

/*
S
Shan Wei 已提交
153
 *	xmit an sk_buff (used by TCP, SCTP and DCCP)
L
Linus Torvalds 已提交
154 155
 */

156
int ip6_xmit(struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
157
	     struct ipv6_txoptions *opt, int tclass)
L
Linus Torvalds 已提交
158
{
159
	struct net *net = sock_net(sk);
160
	struct ipv6_pinfo *np = inet6_sk(sk);
161
	struct in6_addr *first_hop = &fl6->daddr;
E
Eric Dumazet 已提交
162
	struct dst_entry *dst = skb_dst(skb);
L
Linus Torvalds 已提交
163
	struct ipv6hdr *hdr;
164
	u8  proto = fl6->flowi6_proto;
L
Linus Torvalds 已提交
165
	int seg_len = skb->len;
G
Gerrit Renker 已提交
166
	int hlimit = -1;
L
Linus Torvalds 已提交
167 168 169
	u32 mtu;

	if (opt) {
170
		unsigned int head_room;
L
Linus Torvalds 已提交
171 172 173 174 175 176 177 178 179 180

		/* First: exthdrs may take lots of space (~8K for now)
		   MAX_HEADER is not enough.
		 */
		head_room = opt->opt_nflen + opt->opt_flen;
		seg_len += head_room;
		head_room += sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev);

		if (skb_headroom(skb) < head_room) {
			struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room);
181
			if (!skb2) {
E
Eric Dumazet 已提交
182
				IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
183 184
					      IPSTATS_MIB_OUTDISCARDS);
				kfree_skb(skb);
L
Linus Torvalds 已提交
185 186
				return -ENOBUFS;
			}
187
			consume_skb(skb);
188
			skb = skb2;
189
			skb_set_owner_w(skb, sk);
L
Linus Torvalds 已提交
190 191 192 193 194 195 196
		}
		if (opt->opt_flen)
			ipv6_push_frag_opts(skb, opt, &proto);
		if (opt->opt_nflen)
			ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop);
	}

197 198
	skb_push(skb, sizeof(struct ipv6hdr));
	skb_reset_network_header(skb);
199
	hdr = ipv6_hdr(skb);
L
Linus Torvalds 已提交
200 201 202 203

	/*
	 *	Fill in the IPv6 header
	 */
204
	if (np)
L
Linus Torvalds 已提交
205 206
		hlimit = np->hop_limit;
	if (hlimit < 0)
207
		hlimit = ip6_dst_hoplimit(dst);
L
Linus Torvalds 已提交
208

209
	ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel,
210
						     np->autoflowlabel, fl6));
211

L
Linus Torvalds 已提交
212 213 214 215
	hdr->payload_len = htons(seg_len);
	hdr->nexthdr = proto;
	hdr->hop_limit = hlimit;

A
Alexey Dobriyan 已提交
216 217
	hdr->saddr = fl6->saddr;
	hdr->daddr = *first_hop;
L
Linus Torvalds 已提交
218

219
	skb->protocol = htons(ETH_P_IPV6);
220
	skb->priority = sk->sk_priority;
221
	skb->mark = sk->sk_mark;
222

L
Linus Torvalds 已提交
223
	mtu = dst_mtu(dst);
W
WANG Cong 已提交
224
	if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) {
E
Eric Dumazet 已提交
225
		IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)),
226
			      IPSTATS_MIB_OUT, skb->len);
227 228
		return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, sk, skb,
			       NULL, dst->dev, dst_output_sk);
L
Linus Torvalds 已提交
229 230 231
	}

	skb->dev = dst->dev;
232
	ipv6_local_error(sk, EMSGSIZE, fl6, mtu);
E
Eric Dumazet 已提交
233
	IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
234 235 236
	kfree_skb(skb);
	return -EMSGSIZE;
}
237 238
EXPORT_SYMBOL(ip6_xmit);

L
Linus Torvalds 已提交
239 240 241 242 243 244 245 246
static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
{
	struct ip6_ra_chain *ra;
	struct sock *last = NULL;

	read_lock(&ip6_ra_lock);
	for (ra = ip6_ra_chain; ra; ra = ra->next) {
		struct sock *sk = ra->sk;
247 248 249
		if (sk && ra->sel == sel &&
		    (!sk->sk_bound_dev_if ||
		     sk->sk_bound_dev_if == skb->dev->ifindex)) {
L
Linus Torvalds 已提交
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
			if (last) {
				struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
				if (skb2)
					rawv6_rcv(last, skb2);
			}
			last = sk;
		}
	}

	if (last) {
		rawv6_rcv(last, skb);
		read_unlock(&ip6_ra_lock);
		return 1;
	}
	read_unlock(&ip6_ra_lock);
	return 0;
}

268 269
static int ip6_forward_proxy_check(struct sk_buff *skb)
{
270
	struct ipv6hdr *hdr = ipv6_hdr(skb);
271
	u8 nexthdr = hdr->nexthdr;
272
	__be16 frag_off;
273 274 275
	int offset;

	if (ipv6_ext_hdr(nexthdr)) {
276
		offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
277 278 279 280 281 282 283 284
		if (offset < 0)
			return 0;
	} else
		offset = sizeof(struct ipv6hdr);

	if (nexthdr == IPPROTO_ICMPV6) {
		struct icmp6hdr *icmp6;

285 286
		if (!pskb_may_pull(skb, (skb_network_header(skb) +
					 offset + 1 - skb->data)))
287 288
			return 0;

289
		icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306

		switch (icmp6->icmp6_type) {
		case NDISC_ROUTER_SOLICITATION:
		case NDISC_ROUTER_ADVERTISEMENT:
		case NDISC_NEIGHBOUR_SOLICITATION:
		case NDISC_NEIGHBOUR_ADVERTISEMENT:
		case NDISC_REDIRECT:
			/* For reaction involving unicast neighbor discovery
			 * message destined to the proxied address, pass it to
			 * input function.
			 */
			return 1;
		default:
			break;
		}
	}

307 308 309 310 311 312 313 314 315 316
	/*
	 * The proxying router can't forward traffic sent to a link-local
	 * address, so signal the sender and discard the packet. This
	 * behavior is clarified by the MIPv6 specification.
	 */
	if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
		dst_link_failure(skb);
		return -1;
	}

317 318 319
	return 0;
}

320
static inline int ip6_forward_finish(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
321
{
322
	skb_sender_cpu_clear(skb);
323
	return dst_output_sk(sk, skb);
L
Linus Torvalds 已提交
324 325
}

326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
static unsigned int ip6_dst_mtu_forward(const struct dst_entry *dst)
{
	unsigned int mtu;
	struct inet6_dev *idev;

	if (dst_metric_locked(dst, RTAX_MTU)) {
		mtu = dst_metric_raw(dst, RTAX_MTU);
		if (mtu)
			return mtu;
	}

	mtu = IPV6_MIN_MTU;
	rcu_read_lock();
	idev = __in6_dev_get(dst->dev);
	if (idev)
		mtu = idev->cnf.mtu6;
	rcu_read_unlock();

	return mtu;
}

347 348
static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
{
349
	if (skb->len <= mtu)
350 351
		return false;

W
WANG Cong 已提交
352
	/* ipv6 conntrack defrag sets max_frag_size + ignore_df */
353 354 355
	if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)
		return true;

W
WANG Cong 已提交
356
	if (skb->ignore_df)
357 358
		return false;

359 360 361 362 363 364
	if (skb_is_gso(skb) && skb_gso_network_seglen(skb) <= mtu)
		return false;

	return true;
}

L
Linus Torvalds 已提交
365 366
int ip6_forward(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
367
	struct dst_entry *dst = skb_dst(skb);
368
	struct ipv6hdr *hdr = ipv6_hdr(skb);
L
Linus Torvalds 已提交
369
	struct inet6_skb_parm *opt = IP6CB(skb);
370
	struct net *net = dev_net(dst->dev);
371
	u32 mtu;
372

373
	if (net->ipv6.devconf_all->forwarding == 0)
L
Linus Torvalds 已提交
374 375
		goto error;

376 377 378
	if (skb->pkt_type != PACKET_HOST)
		goto drop;

379 380 381
	if (skb_warn_if_lro(skb))
		goto drop;

L
Linus Torvalds 已提交
382
	if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
383 384
		IP6_INC_STATS_BH(net, ip6_dst_idev(dst),
				 IPSTATS_MIB_INDISCARDS);
L
Linus Torvalds 已提交
385 386 387
		goto drop;
	}

388
	skb_forward_csum(skb);
L
Linus Torvalds 已提交
389 390 391 392 393 394 395 396 397 398 399 400 401 402

	/*
	 *	We DO NOT make any processing on
	 *	RA packets, pushing them to user level AS IS
	 *	without ane WARRANTY that application will be able
	 *	to interpret them. The reason is that we
	 *	cannot make anything clever here.
	 *
	 *	We are not end-node, so that if packet contains
	 *	AH/ESP, we cannot make anything.
	 *	Defragmentation also would be mistake, RA packets
	 *	cannot be fragmented, because there is no warranty
	 *	that different fragments will go along one path. --ANK
	 */
403 404
	if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
		if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
L
Linus Torvalds 已提交
405 406 407 408 409 410 411 412 413
			return 0;
	}

	/*
	 *	check and decrement ttl
	 */
	if (hdr->hop_limit <= 1) {
		/* Force OUTPUT device used as source address */
		skb->dev = dst->dev;
414
		icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
415 416
		IP6_INC_STATS_BH(net, ip6_dst_idev(dst),
				 IPSTATS_MIB_INHDRERRORS);
L
Linus Torvalds 已提交
417 418 419 420 421

		kfree_skb(skb);
		return -ETIMEDOUT;
	}

422
	/* XXX: idev->cnf.proxy_ndp? */
423
	if (net->ipv6.devconf_all->proxy_ndp &&
424
	    pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) {
425 426
		int proxied = ip6_forward_proxy_check(skb);
		if (proxied > 0)
427
			return ip6_input(skb);
428
		else if (proxied < 0) {
429 430
			IP6_INC_STATS_BH(net, ip6_dst_idev(dst),
					 IPSTATS_MIB_INDISCARDS);
431 432
			goto drop;
		}
433 434
	}

L
Linus Torvalds 已提交
435
	if (!xfrm6_route_forward(skb)) {
436 437
		IP6_INC_STATS_BH(net, ip6_dst_idev(dst),
				 IPSTATS_MIB_INDISCARDS);
L
Linus Torvalds 已提交
438 439
		goto drop;
	}
E
Eric Dumazet 已提交
440
	dst = skb_dst(skb);
L
Linus Torvalds 已提交
441 442 443

	/* IPv6 specs say nothing about it, but it is clear that we cannot
	   send redirects to source routed frames.
444
	   We don't send redirects to frames decapsulated from IPsec.
L
Linus Torvalds 已提交
445
	 */
446
	if (skb->dev == dst->dev && opt->srcrt == 0 && !skb_sec_path(skb)) {
L
Linus Torvalds 已提交
447
		struct in6_addr *target = NULL;
448
		struct inet_peer *peer;
L
Linus Torvalds 已提交
449 450 451 452 453 454 455 456
		struct rt6_info *rt;

		/*
		 *	incoming and outgoing devices are the same
		 *	send a redirect.
		 */

		rt = (struct rt6_info *) dst;
457 458
		if (rt->rt6i_flags & RTF_GATEWAY)
			target = &rt->rt6i_gateway;
L
Linus Torvalds 已提交
459 460 461
		else
			target = &hdr->daddr;

462
		peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1);
463

L
Linus Torvalds 已提交
464 465 466
		/* Limit redirects both by destination (here)
		   and by source (inside ndisc_send_redirect)
		 */
467
		if (inet_peer_xrlim_allow(peer, 1*HZ))
468
			ndisc_send_redirect(skb, target);
469 470
		if (peer)
			inet_putpeer(peer);
471 472 473
	} else {
		int addrtype = ipv6_addr_type(&hdr->saddr);

L
Linus Torvalds 已提交
474
		/* This check is security critical. */
475 476
		if (addrtype == IPV6_ADDR_ANY ||
		    addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
477 478 479
			goto error;
		if (addrtype & IPV6_ADDR_LINKLOCAL) {
			icmpv6_send(skb, ICMPV6_DEST_UNREACH,
480
				    ICMPV6_NOT_NEIGHBOUR, 0);
481 482
			goto error;
		}
L
Linus Torvalds 已提交
483 484
	}

485
	mtu = ip6_dst_mtu_forward(dst);
486 487 488
	if (mtu < IPV6_MIN_MTU)
		mtu = IPV6_MIN_MTU;

489
	if (ip6_pkt_too_big(skb, mtu)) {
L
Linus Torvalds 已提交
490 491
		/* Again, force OUTPUT device used as source address */
		skb->dev = dst->dev;
492
		icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
493 494 495 496
		IP6_INC_STATS_BH(net, ip6_dst_idev(dst),
				 IPSTATS_MIB_INTOOBIGERRORS);
		IP6_INC_STATS_BH(net, ip6_dst_idev(dst),
				 IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
497 498 499 500 501
		kfree_skb(skb);
		return -EMSGSIZE;
	}

	if (skb_cow(skb, dst->dev->hard_header_len)) {
502 503
		IP6_INC_STATS_BH(net, ip6_dst_idev(dst),
				 IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
504 505 506
		goto drop;
	}

507
	hdr = ipv6_hdr(skb);
L
Linus Torvalds 已提交
508 509

	/* Mangling hops number delayed to point after skb COW */
510

L
Linus Torvalds 已提交
511 512
	hdr->hop_limit--;

513
	IP6_INC_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
514
	IP6_ADD_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len);
515 516
	return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, NULL, skb,
		       skb->dev, dst->dev,
517
		       ip6_forward_finish);
L
Linus Torvalds 已提交
518 519

error:
520
	IP6_INC_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_INADDRERRORS);
L
Linus Torvalds 已提交
521 522 523 524 525 526 527 528 529 530
drop:
	kfree_skb(skb);
	return -EINVAL;
}

static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
{
	to->pkt_type = from->pkt_type;
	to->priority = from->priority;
	to->protocol = from->protocol;
E
Eric Dumazet 已提交
531 532
	skb_dst_drop(to);
	skb_dst_set(to, dst_clone(skb_dst(from)));
L
Linus Torvalds 已提交
533
	to->dev = from->dev;
T
Thomas Graf 已提交
534
	to->mark = from->mark;
L
Linus Torvalds 已提交
535 536 537 538

#ifdef CONFIG_NET_SCHED
	to->tc_index = from->tc_index;
#endif
539
	nf_copy(to, from);
540
	skb_copy_secmark(to, from);
L
Linus Torvalds 已提交
541 542
}

543 544
int ip6_fragment(struct sock *sk, struct sk_buff *skb,
		 int (*output)(struct sock *, struct sk_buff *))
L
Linus Torvalds 已提交
545 546
{
	struct sk_buff *frag;
547
	struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
548 549
	struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
				inet6_sk(skb->sk) : NULL;
L
Linus Torvalds 已提交
550 551 552
	struct ipv6hdr *tmp_hdr;
	struct frag_hdr *fh;
	unsigned int mtu, hlen, left, len;
553
	int hroom, troom;
554
	__be32 frag_id;
555
	int ptr, offset = 0, err = 0;
L
Linus Torvalds 已提交
556
	u8 *prevhdr, nexthdr = 0;
E
Eric Dumazet 已提交
557
	struct net *net = dev_net(skb_dst(skb)->dev);
L
Linus Torvalds 已提交
558 559 560 561

	hlen = ip6_find_1stfragopt(skb, &prevhdr);
	nexthdr = *prevhdr;

J
John Heffner 已提交
562
	mtu = ip6_skb_dst_mtu(skb);
563 564

	/* We must not fragment if the socket is set to force MTU discovery
565
	 * or if the skb it not generated by a local socket.
566
	 */
567 568
	if (unlikely(!skb->ignore_df && skb->len > mtu))
		goto fail_toobig;
569

570 571 572 573 574 575 576 577
	if (IP6CB(skb)->frag_max_size) {
		if (IP6CB(skb)->frag_max_size > mtu)
			goto fail_toobig;

		/* don't send fragments larger than what we received */
		mtu = IP6CB(skb)->frag_max_size;
		if (mtu < IPV6_MIN_MTU)
			mtu = IPV6_MIN_MTU;
578 579
	}

580 581 582 583 584
	if (np && np->frag_size < mtu) {
		if (np->frag_size)
			mtu = np->frag_size;
	}
	mtu -= hlen + sizeof(struct frag_hdr);
L
Linus Torvalds 已提交
585

586 587
	frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
				    &ipv6_hdr(skb)->saddr);
588

589
	if (skb_has_frag_list(skb)) {
L
Linus Torvalds 已提交
590
		int first_len = skb_pagelen(skb);
591
		struct sk_buff *frag2;
L
Linus Torvalds 已提交
592 593 594 595 596 597

		if (first_len - hlen > mtu ||
		    ((first_len - hlen) & 7) ||
		    skb_cloned(skb))
			goto slow_path;

598
		skb_walk_frags(skb, frag) {
L
Linus Torvalds 已提交
599 600 601 602
			/* Correct geometry. */
			if (frag->len > mtu ||
			    ((frag->len & 7) && frag->next) ||
			    skb_headroom(frag) < hlen)
603
				goto slow_path_clean;
L
Linus Torvalds 已提交
604 605 606

			/* Partially cloned skb? */
			if (skb_shared(frag))
607
				goto slow_path_clean;
608 609 610 611 612 613

			BUG_ON(frag->sk);
			if (skb->sk) {
				frag->sk = skb->sk;
				frag->destructor = sock_wfree;
			}
614
			skb->truesize -= frag->truesize;
L
Linus Torvalds 已提交
615 616 617 618 619
		}

		err = 0;
		offset = 0;
		frag = skb_shinfo(skb)->frag_list;
620
		skb_frag_list_init(skb);
L
Linus Torvalds 已提交
621 622
		/* BUILD HEADER */

Y
YOSHIFUJI Hideaki 已提交
623
		*prevhdr = NEXTHDR_FRAGMENT;
624
		tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
L
Linus Torvalds 已提交
625
		if (!tmp_hdr) {
E
Eric Dumazet 已提交
626
			IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
627
				      IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
628 629 630 631
			return -ENOMEM;
		}

		__skb_pull(skb, hlen);
632
		fh = (struct frag_hdr *)__skb_push(skb, sizeof(struct frag_hdr));
633 634
		__skb_push(skb, hlen);
		skb_reset_network_header(skb);
635
		memcpy(skb_network_header(skb), tmp_hdr, hlen);
L
Linus Torvalds 已提交
636 637 638 639

		fh->nexthdr = nexthdr;
		fh->reserved = 0;
		fh->frag_off = htons(IP6_MF);
640
		fh->identification = frag_id;
L
Linus Torvalds 已提交
641 642 643 644

		first_len = skb_pagelen(skb);
		skb->data_len = first_len - skb_headlen(skb);
		skb->len = first_len;
645 646
		ipv6_hdr(skb)->payload_len = htons(first_len -
						   sizeof(struct ipv6hdr));
647

648
		dst_hold(&rt->dst);
L
Linus Torvalds 已提交
649 650 651 652 653 654

		for (;;) {
			/* Prepare header of the next frame,
			 * before previous one went down. */
			if (frag) {
				frag->ip_summed = CHECKSUM_NONE;
655
				skb_reset_transport_header(frag);
656
				fh = (struct frag_hdr *)__skb_push(frag, sizeof(struct frag_hdr));
657 658
				__skb_push(frag, hlen);
				skb_reset_network_header(frag);
659 660
				memcpy(skb_network_header(frag), tmp_hdr,
				       hlen);
L
Linus Torvalds 已提交
661 662 663 664
				offset += skb->len - hlen - sizeof(struct frag_hdr);
				fh->nexthdr = nexthdr;
				fh->reserved = 0;
				fh->frag_off = htons(offset);
665
				if (frag->next)
L
Linus Torvalds 已提交
666 667
					fh->frag_off |= htons(IP6_MF);
				fh->identification = frag_id;
668 669 670
				ipv6_hdr(frag)->payload_len =
						htons(frag->len -
						      sizeof(struct ipv6hdr));
L
Linus Torvalds 已提交
671 672
				ip6_copy_metadata(frag, skb);
			}
673

674
			err = output(sk, skb);
675
			if (!err)
676
				IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
677
					      IPSTATS_MIB_FRAGCREATES);
678

L
Linus Torvalds 已提交
679 680 681 682 683 684 685 686
			if (err || !frag)
				break;

			skb = frag;
			frag = skb->next;
			skb->next = NULL;
		}

J
Jesper Juhl 已提交
687
		kfree(tmp_hdr);
L
Linus Torvalds 已提交
688 689

		if (err == 0) {
690
			IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
691
				      IPSTATS_MIB_FRAGOKS);
A
Amerigo Wang 已提交
692
			ip6_rt_put(rt);
L
Linus Torvalds 已提交
693 694 695
			return 0;
		}

696
		kfree_skb_list(frag);
L
Linus Torvalds 已提交
697

698
		IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
699
			      IPSTATS_MIB_FRAGFAILS);
A
Amerigo Wang 已提交
700
		ip6_rt_put(rt);
L
Linus Torvalds 已提交
701
		return err;
702 703 704 705 706 707 708 709 710

slow_path_clean:
		skb_walk_frags(skb, frag2) {
			if (frag2 == frag)
				break;
			frag2->sk = NULL;
			frag2->destructor = NULL;
			skb->truesize += frag2->truesize;
		}
L
Linus Torvalds 已提交
711 712 713
	}

slow_path:
714 715 716 717
	if ((skb->ip_summed == CHECKSUM_PARTIAL) &&
	    skb_checksum_help(skb))
		goto fail;

L
Linus Torvalds 已提交
718 719 720 721 722 723 724 725
	left = skb->len - hlen;		/* Space per frame */
	ptr = hlen;			/* Where to start from */

	/*
	 *	Fragment the datagram.
	 */

	*prevhdr = NEXTHDR_FRAGMENT;
726 727
	hroom = LL_RESERVED_SPACE(rt->dst.dev);
	troom = rt->dst.dev->needed_tailroom;
L
Linus Torvalds 已提交
728 729 730 731

	/*
	 *	Keep copying data until we run out.
	 */
732
	while (left > 0)	{
L
Linus Torvalds 已提交
733 734 735 736
		len = left;
		/* IF: it doesn't fit, use 'mtu' - the data space left */
		if (len > mtu)
			len = mtu;
L
Lucas De Marchi 已提交
737
		/* IF: we are not sending up to and including the packet end
L
Linus Torvalds 已提交
738 739 740 741 742
		   then align the next start on an eight byte boundary */
		if (len < left)	{
			len &= ~7;
		}

743 744 745 746
		/* Allocate buffer */
		frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) +
				 hroom + troom, GFP_ATOMIC);
		if (!frag) {
E
Eric Dumazet 已提交
747
			IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
748
				      IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
749 750 751 752 753 754 755 756 757
			err = -ENOMEM;
			goto fail;
		}

		/*
		 *	Set up data on packet
		 */

		ip6_copy_metadata(frag, skb);
758
		skb_reserve(frag, hroom);
L
Linus Torvalds 已提交
759
		skb_put(frag, len + hlen + sizeof(struct frag_hdr));
760
		skb_reset_network_header(frag);
761
		fh = (struct frag_hdr *)(skb_network_header(frag) + hlen);
762 763
		frag->transport_header = (frag->network_header + hlen +
					  sizeof(struct frag_hdr));
L
Linus Torvalds 已提交
764 765 766 767 768 769 770 771 772 773 774

		/*
		 *	Charge the memory for the fragment to any owner
		 *	it might possess
		 */
		if (skb->sk)
			skb_set_owner_w(frag, skb->sk);

		/*
		 *	Copy the packet header into the new buffer.
		 */
775
		skb_copy_from_linear_data(skb, skb_network_header(frag), hlen);
L
Linus Torvalds 已提交
776 777 778 779 780 781

		/*
		 *	Build fragment header.
		 */
		fh->nexthdr = nexthdr;
		fh->reserved = 0;
782
		fh->identification = frag_id;
L
Linus Torvalds 已提交
783 784 785 786

		/*
		 *	Copy a block of the IP datagram.
		 */
H
Himangi Saraogi 已提交
787 788
		BUG_ON(skb_copy_bits(skb, ptr, skb_transport_header(frag),
				     len));
L
Linus Torvalds 已提交
789 790 791 792 793
		left -= len;

		fh->frag_off = htons(offset);
		if (left > 0)
			fh->frag_off |= htons(IP6_MF);
794 795
		ipv6_hdr(frag)->payload_len = htons(frag->len -
						    sizeof(struct ipv6hdr));
L
Linus Torvalds 已提交
796 797 798 799 800 801 802

		ptr += len;
		offset += len;

		/*
		 *	Put this fragment into the sending queue.
		 */
803
		err = output(sk, frag);
L
Linus Torvalds 已提交
804 805
		if (err)
			goto fail;
806

E
Eric Dumazet 已提交
807
		IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
808
			      IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
809
	}
E
Eric Dumazet 已提交
810
	IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
811
		      IPSTATS_MIB_FRAGOKS);
812
	consume_skb(skb);
L
Linus Torvalds 已提交
813 814
	return err;

815 816 817 818 819 820 821 822
fail_toobig:
	if (skb->sk && dst_allfrag(skb_dst(skb)))
		sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK);

	skb->dev = skb_dst(skb)->dev;
	icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
	err = -EMSGSIZE;

L
Linus Torvalds 已提交
823
fail:
E
Eric Dumazet 已提交
824
	IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
825
		      IPSTATS_MIB_FRAGFAILS);
826
	kfree_skb(skb);
L
Linus Torvalds 已提交
827 828 829
	return err;
}

830 831 832
static inline int ip6_rt_check(const struct rt6key *rt_key,
			       const struct in6_addr *fl_addr,
			       const struct in6_addr *addr_cache)
833
{
E
Eric Dumazet 已提交
834
	return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
835
		(!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache));
836 837
}

838 839
static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
					  struct dst_entry *dst,
840
					  const struct flowi6 *fl6)
L
Linus Torvalds 已提交
841
{
842
	struct ipv6_pinfo *np = inet6_sk(sk);
843
	struct rt6_info *rt;
L
Linus Torvalds 已提交
844

845 846 847
	if (!dst)
		goto out;

848 849 850 851 852 853
	if (dst->ops->family != AF_INET6) {
		dst_release(dst);
		return NULL;
	}

	rt = (struct rt6_info *)dst;
854 855 856
	/* Yes, checking route validity in not connected
	 * case is not very simple. Take into account,
	 * that we do not support routing by source, TOS,
857
	 * and MSG_DONTROUTE		--ANK (980726)
858
	 *
859 860
	 * 1. ip6_rt_check(): If route was host route,
	 *    check that cached destination is current.
861 862 863 864 865 866 867 868 869 870
	 *    If it is network route, we still may
	 *    check its validity using saved pointer
	 *    to the last used address: daddr_cache.
	 *    We do not want to save whole address now,
	 *    (because main consumer of this service
	 *    is tcp, which has not this problem),
	 *    so that the last trick works only on connected
	 *    sockets.
	 * 2. oif also should be the same.
	 */
871
	if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) ||
872
#ifdef CONFIG_IPV6_SUBTREES
873
	    ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) ||
874
#endif
875
	    (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex)) {
876 877
		dst_release(dst);
		dst = NULL;
L
Linus Torvalds 已提交
878 879
	}

880 881 882 883
out:
	return dst;
}

884
static int ip6_dst_lookup_tail(struct net *net, struct sock *sk,
885
			       struct dst_entry **dst, struct flowi6 *fl6)
886
{
887 888
#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
	struct neighbour *n;
889
	struct rt6_info *rt;
890 891
#endif
	int err;
892

893 894 895 896 897 898 899 900 901 902 903 904
	/* The correct way to handle this would be to do
	 * ip6_route_get_saddr, and then ip6_route_output; however,
	 * the route-specific preferred source forces the
	 * ip6_route_output call _before_ ip6_route_get_saddr.
	 *
	 * In source specific routing (no src=any default route),
	 * ip6_route_output will fail given src=any saddr, though, so
	 * that's why we try it again later.
	 */
	if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) {
		struct rt6_info *rt;
		bool had_dst = *dst != NULL;
L
Linus Torvalds 已提交
905

906 907 908
		if (!had_dst)
			*dst = ip6_route_output(net, sk, fl6);
		rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
909 910 911
		err = ip6_route_get_saddr(net, rt, &fl6->daddr,
					  sk ? inet6_sk(sk)->srcprefs : 0,
					  &fl6->saddr);
912
		if (err)
L
Linus Torvalds 已提交
913
			goto out_err_release;
914 915 916 917 918 919 920 921 922

		/* If we had an erroneous initial result, pretend it
		 * never existed and let the SA-enabled version take
		 * over.
		 */
		if (!had_dst && (*dst)->error) {
			dst_release(*dst);
			*dst = NULL;
		}
L
Linus Torvalds 已提交
923 924
	}

925 926 927 928 929 930 931
	if (!*dst)
		*dst = ip6_route_output(net, sk, fl6);

	err = (*dst)->error;
	if (err)
		goto out_err_release;

932
#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
933 934 935 936 937 938 939 940
	/*
	 * Here if the dst entry we've looked up
	 * has a neighbour entry that is in the INCOMPLETE
	 * state and the src address from the flow is
	 * marked as OPTIMISTIC, we release the found
	 * dst entry and replace it instead with the
	 * dst entry of the nexthop router
	 */
941
	rt = (struct rt6_info *) *dst;
942
	rcu_read_lock_bh();
943 944
	n = __ipv6_neigh_lookup_noref(rt->dst.dev,
				      rt6_nexthop(rt, &fl6->daddr));
945 946 947 948
	err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0;
	rcu_read_unlock_bh();

	if (err) {
949
		struct inet6_ifaddr *ifp;
950
		struct flowi6 fl_gw6;
951 952
		int redirect;

953
		ifp = ipv6_get_ifaddr(net, &fl6->saddr,
954 955 956 957 958 959 960 961 962 963 964 965
				      (*dst)->dev, 1);

		redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
		if (ifp)
			in6_ifa_put(ifp);

		if (redirect) {
			/*
			 * We need to get the dst entry for the
			 * default router instead
			 */
			dst_release(*dst);
966 967 968
			memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
			memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
			*dst = ip6_route_output(net, sk, &fl_gw6);
969 970
			err = (*dst)->error;
			if (err)
971
				goto out_err_release;
972
		}
973
	}
974 975
#endif

L
Linus Torvalds 已提交
976 977 978
	return 0;

out_err_release:
979
	if (err == -ENETUNREACH)
980
		IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
L
Linus Torvalds 已提交
981 982 983 984
	dst_release(*dst);
	*dst = NULL;
	return err;
}
A
Adrian Bunk 已提交
985

986 987 988 989
/**
 *	ip6_dst_lookup - perform route lookup on flow
 *	@sk: socket which provides route info
 *	@dst: pointer to dst_entry * for result
990
 *	@fl6: flow to lookup
991 992 993 994 995
 *
 *	This function performs a route lookup on the given flow.
 *
 *	It returns zero on success, or a standard errno code on error.
 */
996 997
int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst,
		   struct flowi6 *fl6)
998 999
{
	*dst = NULL;
1000
	return ip6_dst_lookup_tail(net, sk, dst, fl6);
1001
}
1002 1003
EXPORT_SYMBOL_GPL(ip6_dst_lookup);

1004
/**
1005 1006
 *	ip6_dst_lookup_flow - perform route lookup on flow with ipsec
 *	@sk: socket which provides route info
1007
 *	@fl6: flow to lookup
1008 1009 1010 1011 1012 1013 1014
 *	@final_dst: final destination address for ipsec lookup
 *
 *	This function performs a route lookup on the given flow.
 *
 *	It returns a valid dst pointer on success, or a pointer encoded
 *	error code.
 */
1015
struct dst_entry *ip6_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1016
				      const struct in6_addr *final_dst)
1017 1018 1019 1020
{
	struct dst_entry *dst = NULL;
	int err;

1021
	err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6);
1022 1023 1024
	if (err)
		return ERR_PTR(err);
	if (final_dst)
A
Alexey Dobriyan 已提交
1025
		fl6->daddr = *final_dst;
1026 1027
	if (!fl6->flowi6_oif)
		fl6->flowi6_oif = dst->dev->ifindex;
1028

1029
	return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0);
1030 1031 1032 1033 1034
}
EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);

/**
 *	ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1035
 *	@sk: socket which provides the dst cache and route info
1036
 *	@fl6: flow to lookup
1037
 *	@final_dst: final destination address for ipsec lookup
1038 1039 1040 1041 1042 1043
 *
 *	This function performs a route lookup on the given flow with the
 *	possibility of using the cached route in the socket if it is valid.
 *	It will take the socket dst lock when operating on the dst cache.
 *	As a result, this function can only be used in process context.
 *
1044 1045
 *	It returns a valid dst pointer on success, or a pointer encoded
 *	error code.
1046
 */
1047
struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1048
					 const struct in6_addr *final_dst)
1049
{
1050 1051
	struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
	int err;
1052

1053
	dst = ip6_sk_dst_check(sk, dst, fl6);
1054

1055
	err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6);
1056 1057 1058
	if (err)
		return ERR_PTR(err);
	if (final_dst)
A
Alexey Dobriyan 已提交
1059
		fl6->daddr = *final_dst;
1060

1061
	return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0);
1062
}
1063
EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow);
1064

A
Adrian Bunk 已提交
1065
static inline int ip6_ufo_append_data(struct sock *sk,
1066
			struct sk_buff_head *queue,
1067 1068 1069
			int getfrag(void *from, char *to, int offset, int len,
			int odd, struct sk_buff *skb),
			void *from, int length, int hh_len, int fragheaderlen,
1070
			int transhdrlen, int mtu, unsigned int flags,
1071
			const struct flowi6 *fl6)
1072 1073 1074 1075 1076 1077 1078 1079 1080

{
	struct sk_buff *skb;
	int err;

	/* There is support for UDP large send offload by network
	 * device, so create one single skb packet containing complete
	 * udp datagram
	 */
1081
	skb = skb_peek_tail(queue);
1082
	if (!skb) {
1083 1084 1085
		skb = sock_alloc_send_skb(sk,
			hh_len + fragheaderlen + transhdrlen + 20,
			(flags & MSG_DONTWAIT), &err);
1086
		if (!skb)
1087
			return err;
1088 1089 1090 1091 1092

		/* reserve space for Hardware header */
		skb_reserve(skb, hh_len);

		/* create space for UDP/IP header */
1093
		skb_put(skb, fragheaderlen + transhdrlen);
1094 1095

		/* initialize network header pointer */
1096
		skb_reset_network_header(skb);
1097 1098

		/* initialize protocol header pointer */
1099
		skb->transport_header = skb->network_header + fragheaderlen;
1100

1101
		skb->protocol = htons(ETH_P_IPV6);
1102 1103
		skb->csum = 0;

1104
		__skb_queue_tail(queue, skb);
1105 1106
	} else if (skb_is_gso(skb)) {
		goto append;
1107 1108
	}

1109 1110 1111 1112 1113 1114 1115
	skb->ip_summed = CHECKSUM_PARTIAL;
	/* Specify the length of each IPv6 datagram fragment.
	 * It has to be a multiple of 8.
	 */
	skb_shinfo(skb)->gso_size = (mtu - fragheaderlen -
				     sizeof(struct frag_hdr)) & ~7;
	skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1116 1117 1118
	skb_shinfo(skb)->ip6_frag_id = ipv6_select_ident(sock_net(sk),
							 &fl6->daddr,
							 &fl6->saddr);
1119 1120

append:
1121 1122
	return skb_append_datato_frags(sk, skb, getfrag, from,
				       (length - transhdrlen));
1123
}
L
Linus Torvalds 已提交
1124

1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
					       gfp_t gfp)
{
	return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
}

static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
						gfp_t gfp)
{
	return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
}

1137
static void ip6_append_data_mtu(unsigned int *mtu,
G
Gao feng 已提交
1138 1139 1140
				int *maxfraglen,
				unsigned int fragheaderlen,
				struct sk_buff *skb,
1141
				struct rt6_info *rt,
1142
				unsigned int orig_mtu)
G
Gao feng 已提交
1143 1144
{
	if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
1145
		if (!skb) {
G
Gao feng 已提交
1146
			/* first fragment, reserve header_len */
1147
			*mtu = orig_mtu - rt->dst.header_len;
G
Gao feng 已提交
1148 1149 1150 1151 1152 1153

		} else {
			/*
			 * this fragment is not first, the headers
			 * space is regarded as data space.
			 */
1154
			*mtu = orig_mtu;
G
Gao feng 已提交
1155 1156 1157 1158 1159 1160
		}
		*maxfraglen = ((*mtu - fragheaderlen) & ~7)
			      + fragheaderlen - sizeof(struct frag_hdr);
	}
}

1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
			  struct inet6_cork *v6_cork,
			  int hlimit, int tclass, struct ipv6_txoptions *opt,
			  struct rt6_info *rt, struct flowi6 *fl6)
{
	struct ipv6_pinfo *np = inet6_sk(sk);
	unsigned int mtu;

	/*
	 * setup for corking
	 */
	if (opt) {
		if (WARN_ON(v6_cork->opt))
			return -EINVAL;

		v6_cork->opt = kzalloc(opt->tot_len, sk->sk_allocation);
1177
		if (unlikely(!v6_cork->opt))
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
			return -ENOBUFS;

		v6_cork->opt->tot_len = opt->tot_len;
		v6_cork->opt->opt_flen = opt->opt_flen;
		v6_cork->opt->opt_nflen = opt->opt_nflen;

		v6_cork->opt->dst0opt = ip6_opt_dup(opt->dst0opt,
						    sk->sk_allocation);
		if (opt->dst0opt && !v6_cork->opt->dst0opt)
			return -ENOBUFS;

		v6_cork->opt->dst1opt = ip6_opt_dup(opt->dst1opt,
						    sk->sk_allocation);
		if (opt->dst1opt && !v6_cork->opt->dst1opt)
			return -ENOBUFS;

		v6_cork->opt->hopopt = ip6_opt_dup(opt->hopopt,
						   sk->sk_allocation);
		if (opt->hopopt && !v6_cork->opt->hopopt)
			return -ENOBUFS;

		v6_cork->opt->srcrt = ip6_rthdr_dup(opt->srcrt,
						    sk->sk_allocation);
		if (opt->srcrt && !v6_cork->opt->srcrt)
			return -ENOBUFS;

		/* need source address above miyazawa*/
	}
	dst_hold(&rt->dst);
	cork->base.dst = &rt->dst;
	cork->fl.u.ip6 = *fl6;
	v6_cork->hop_limit = hlimit;
	v6_cork->tclass = tclass;
	if (rt->dst.flags & DST_XFRM_TUNNEL)
		mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
		      rt->dst.dev->mtu : dst_mtu(&rt->dst);
	else
		mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
		      rt->dst.dev->mtu : dst_mtu(rt->dst.path);
	if (np->frag_size < mtu) {
		if (np->frag_size)
			mtu = np->frag_size;
	}
	cork->base.fragsize = mtu;
	if (dst_allfrag(rt->dst.path))
		cork->base.flags |= IPCORK_ALLFRAG;
	cork->base.length = 0;

	return 0;
}

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
static int __ip6_append_data(struct sock *sk,
			     struct flowi6 *fl6,
			     struct sk_buff_head *queue,
			     struct inet_cork *cork,
			     struct inet6_cork *v6_cork,
			     struct page_frag *pfrag,
			     int getfrag(void *from, char *to, int offset,
					 int len, int odd, struct sk_buff *skb),
			     void *from, int length, int transhdrlen,
			     unsigned int flags, int dontfrag)
L
Linus Torvalds 已提交
1239
{
G
Gao feng 已提交
1240
	struct sk_buff *skb, *skb_prev = NULL;
1241
	unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu;
1242 1243
	int exthdrlen = 0;
	int dst_exthdrlen = 0;
L
Linus Torvalds 已提交
1244 1245 1246 1247
	int hh_len;
	int copy;
	int err;
	int offset = 0;
1248
	__u8 tx_flags = 0;
1249
	u32 tskey = 0;
1250 1251
	struct rt6_info *rt = (struct rt6_info *)cork->dst;
	struct ipv6_txoptions *opt = v6_cork->opt;
1252
	int csummode = CHECKSUM_NONE;
L
Linus Torvalds 已提交
1253

1254 1255 1256
	skb = skb_peek_tail(queue);
	if (!skb) {
		exthdrlen = opt ? opt->opt_flen : 0;
1257
		dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
L
Linus Torvalds 已提交
1258
	}
1259

1260
	mtu = cork->fragsize;
1261
	orig_mtu = mtu;
L
Linus Torvalds 已提交
1262

1263
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
L
Linus Torvalds 已提交
1264

1265
	fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
1266
			(opt ? opt->opt_nflen : 0);
1267 1268
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
		     sizeof(struct frag_hdr);
L
Linus Torvalds 已提交
1269 1270

	if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) {
1271 1272 1273
		unsigned int maxnonfragsize, headersize;

		headersize = sizeof(struct ipv6hdr) +
1274
			     (opt ? opt->opt_flen + opt->opt_nflen : 0) +
1275 1276 1277 1278
			     (dst_allfrag(&rt->dst) ?
			      sizeof(struct frag_hdr) : 0) +
			     rt->rt6i_nfheader_len;

W
WANG Cong 已提交
1279
		if (ip6_sk_ignore_df(sk))
1280 1281 1282
			maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN;
		else
			maxnonfragsize = mtu;
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297

		/* dontfrag active */
		if ((cork->length + length > mtu - headersize) && dontfrag &&
		    (sk->sk_protocol == IPPROTO_UDP ||
		     sk->sk_protocol == IPPROTO_RAW)) {
			ipv6_local_rxpmtu(sk, fl6, mtu - headersize +
						   sizeof(struct ipv6hdr));
			goto emsgsize;
		}

		if (cork->length + length > maxnonfragsize - headersize) {
emsgsize:
			ipv6_local_error(sk, EMSGSIZE, fl6,
					 mtu - headersize +
					 sizeof(struct ipv6hdr));
L
Linus Torvalds 已提交
1298 1299 1300 1301
			return -EMSGSIZE;
		}
	}

1302
	if (sk->sk_type == SOCK_DGRAM || sk->sk_type == SOCK_RAW) {
1303
		sock_tx_timestamp(sk, &tx_flags);
1304 1305 1306 1307
		if (tx_flags & SKBTX_ANY_SW_TSTAMP &&
		    sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
			tskey = sk->sk_tskey++;
	}
1308

1309 1310
	/* If this is the first and only packet and device
	 * supports checksum offloading, let's use it.
1311 1312
	 * Use transhdrlen, same as IPv4, because partial
	 * sums only work when transhdrlen is set.
1313
	 */
1314
	if (transhdrlen && sk->sk_protocol == IPPROTO_UDP &&
1315 1316 1317 1318
	    length + fragheaderlen < mtu &&
	    rt->dst.dev->features & NETIF_F_V6_CSUM &&
	    !exthdrlen)
		csummode = CHECKSUM_PARTIAL;
L
Linus Torvalds 已提交
1319 1320 1321 1322 1323 1324 1325
	/*
	 * Let's try using as much space as possible.
	 * Use MTU if total length of the message fits into the MTU.
	 * Otherwise, we need to reserve fragment header and
	 * fragment alignment (= 8-15 octects, in total).
	 *
	 * Note that we may need to "move" the data from the tail of
1326
	 * of the buffer to the new fragment when we split
L
Linus Torvalds 已提交
1327 1328
	 * the message.
	 *
1329
	 * FIXME: It may be fragmented into multiple chunks
L
Linus Torvalds 已提交
1330 1331
	 *        at once if non-fragmentable extension headers
	 *        are too large.
1332
	 * --yoshfuji
L
Linus Torvalds 已提交
1333 1334
	 */

1335 1336 1337 1338
	cork->length += length;
	if (((length > mtu) ||
	     (skb && skb_is_gso(skb))) &&
	    (sk->sk_protocol == IPPROTO_UDP) &&
1339 1340
	    (rt->dst.dev->features & NETIF_F_UFO) &&
	    (sk->sk_type == SOCK_DGRAM)) {
1341
		err = ip6_ufo_append_data(sk, queue, getfrag, from, length,
1342
					  hh_len, fragheaderlen,
1343
					  transhdrlen, mtu, flags, fl6);
1344 1345 1346
		if (err)
			goto error;
		return 0;
1347
	}
L
Linus Torvalds 已提交
1348

1349
	if (!skb)
L
Linus Torvalds 已提交
1350 1351 1352 1353
		goto alloc_new_skb;

	while (length > 0) {
		/* Check if the remaining data fits into current packet. */
1354
		copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
L
Linus Torvalds 已提交
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
		if (copy < length)
			copy = maxfraglen - skb->len;

		if (copy <= 0) {
			char *data;
			unsigned int datalen;
			unsigned int fraglen;
			unsigned int fraggap;
			unsigned int alloclen;
alloc_new_skb:
			/* There's no room in the current skb */
G
Gao feng 已提交
1366 1367
			if (skb)
				fraggap = skb->len - maxfraglen;
L
Linus Torvalds 已提交
1368 1369
			else
				fraggap = 0;
G
Gao feng 已提交
1370
			/* update mtu and maxfraglen if necessary */
1371
			if (!skb || !skb_prev)
G
Gao feng 已提交
1372
				ip6_append_data_mtu(&mtu, &maxfraglen,
1373
						    fragheaderlen, skb, rt,
1374
						    orig_mtu);
G
Gao feng 已提交
1375 1376

			skb_prev = skb;
L
Linus Torvalds 已提交
1377 1378 1379 1380 1381 1382 1383

			/*
			 * If remaining data exceeds the mtu,
			 * we know we need more fragment(s).
			 */
			datalen = length + fraggap;

G
Gao feng 已提交
1384 1385
			if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
				datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
L
Linus Torvalds 已提交
1386
			if ((flags & MSG_MORE) &&
1387
			    !(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
1388 1389 1390 1391
				alloclen = mtu;
			else
				alloclen = datalen + fragheaderlen;

1392 1393
			alloclen += dst_exthdrlen;

G
Gao feng 已提交
1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
			if (datalen != length + fraggap) {
				/*
				 * this is not the last fragment, the trailer
				 * space is regarded as data space.
				 */
				datalen += rt->dst.trailer_len;
			}

			alloclen += rt->dst.trailer_len;
			fraglen = datalen + fragheaderlen;
L
Linus Torvalds 已提交
1404 1405 1406

			/*
			 * We just reserve space for fragment header.
1407
			 * Note: this may be overallocation if the message
L
Linus Torvalds 已提交
1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422
			 * (without MSG_MORE) fits into the MTU.
			 */
			alloclen += sizeof(struct frag_hdr);

			if (transhdrlen) {
				skb = sock_alloc_send_skb(sk,
						alloclen + hh_len,
						(flags & MSG_DONTWAIT), &err);
			} else {
				skb = NULL;
				if (atomic_read(&sk->sk_wmem_alloc) <=
				    2 * sk->sk_sndbuf)
					skb = sock_wmalloc(sk,
							   alloclen + hh_len, 1,
							   sk->sk_allocation);
1423
				if (unlikely(!skb))
L
Linus Torvalds 已提交
1424 1425
					err = -ENOBUFS;
			}
1426
			if (!skb)
L
Linus Torvalds 已提交
1427 1428 1429 1430
				goto error;
			/*
			 *	Fill in the control structures
			 */
1431
			skb->protocol = htons(ETH_P_IPV6);
1432
			skb->ip_summed = csummode;
L
Linus Torvalds 已提交
1433
			skb->csum = 0;
1434 1435 1436
			/* reserve for fragmentation and ipsec header */
			skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
				    dst_exthdrlen);
L
Linus Torvalds 已提交
1437

1438 1439 1440
			/* Only the initial fragment is time stamped */
			skb_shinfo(skb)->tx_flags = tx_flags;
			tx_flags = 0;
1441 1442
			skb_shinfo(skb)->tskey = tskey;
			tskey = 0;
1443

L
Linus Torvalds 已提交
1444 1445 1446
			/*
			 *	Find where to start putting bytes
			 */
1447 1448 1449
			data = skb_put(skb, fraglen);
			skb_set_network_header(skb, exthdrlen);
			data += fragheaderlen;
1450 1451
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
L
Linus Torvalds 已提交
1452 1453 1454 1455 1456 1457 1458
			if (fraggap) {
				skb->csum = skb_copy_and_csum_bits(
					skb_prev, maxfraglen,
					data + transhdrlen, fraggap, 0);
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
				data += fraggap;
1459
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1460 1461
			}
			copy = datalen - transhdrlen - fraggap;
1462

L
Linus Torvalds 已提交
1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
			if (copy < 0) {
				err = -EINVAL;
				kfree_skb(skb);
				goto error;
			} else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
				err = -EFAULT;
				kfree_skb(skb);
				goto error;
			}

			offset += copy;
			length -= datalen - fraggap;
			transhdrlen = 0;
			exthdrlen = 0;
1477
			dst_exthdrlen = 0;
L
Linus Torvalds 已提交
1478 1479 1480 1481

			/*
			 * Put the packet on the pending queue
			 */
1482
			__skb_queue_tail(queue, skb);
L
Linus Torvalds 已提交
1483 1484 1485 1486 1487 1488
			continue;
		}

		if (copy > length)
			copy = length;

1489
		if (!(rt->dst.dev->features&NETIF_F_SG)) {
L
Linus Torvalds 已提交
1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
			unsigned int off;

			off = skb->len;
			if (getfrag(from, skb_put(skb, copy),
						offset, copy, off, skb) < 0) {
				__skb_trim(skb, off);
				err = -EFAULT;
				goto error;
			}
		} else {
			int i = skb_shinfo(skb)->nr_frags;

1502 1503
			err = -ENOMEM;
			if (!sk_page_frag_refill(sk, pfrag))
L
Linus Torvalds 已提交
1504
				goto error;
1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515

			if (!skb_can_coalesce(skb, i, pfrag->page,
					      pfrag->offset)) {
				err = -EMSGSIZE;
				if (i == MAX_SKB_FRAGS)
					goto error;

				__skb_fill_page_desc(skb, i, pfrag->page,
						     pfrag->offset, 0);
				skb_shinfo(skb)->nr_frags = ++i;
				get_page(pfrag->page);
L
Linus Torvalds 已提交
1516
			}
1517
			copy = min_t(int, copy, pfrag->size - pfrag->offset);
E
Eric Dumazet 已提交
1518
			if (getfrag(from,
1519 1520 1521 1522 1523 1524
				    page_address(pfrag->page) + pfrag->offset,
				    offset, copy, skb->len, skb) < 0)
				goto error_efault;

			pfrag->offset += copy;
			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
L
Linus Torvalds 已提交
1525 1526
			skb->len += copy;
			skb->data_len += copy;
1527 1528
			skb->truesize += copy;
			atomic_add(copy, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1529 1530 1531 1532
		}
		offset += copy;
		length -= copy;
	}
1533

L
Linus Torvalds 已提交
1534
	return 0;
1535 1536 1537

error_efault:
	err = -EFAULT;
L
Linus Torvalds 已提交
1538
error:
1539
	cork->length -= length;
1540
	IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1541 1542
	return err;
}
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578

int ip6_append_data(struct sock *sk,
		    int getfrag(void *from, char *to, int offset, int len,
				int odd, struct sk_buff *skb),
		    void *from, int length, int transhdrlen, int hlimit,
		    int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6,
		    struct rt6_info *rt, unsigned int flags, int dontfrag)
{
	struct inet_sock *inet = inet_sk(sk);
	struct ipv6_pinfo *np = inet6_sk(sk);
	int exthdrlen;
	int err;

	if (flags&MSG_PROBE)
		return 0;
	if (skb_queue_empty(&sk->sk_write_queue)) {
		/*
		 * setup for corking
		 */
		err = ip6_setup_cork(sk, &inet->cork, &np->cork, hlimit,
				     tclass, opt, rt, fl6);
		if (err)
			return err;

		exthdrlen = (opt ? opt->opt_flen : 0);
		length += exthdrlen;
		transhdrlen += exthdrlen;
	} else {
		fl6 = &inet->cork.fl.u.ip6;
		transhdrlen = 0;
	}

	return __ip6_append_data(sk, fl6, &sk->sk_write_queue, &inet->cork.base,
				 &np->cork, sk_page_frag(sk), getfrag,
				 from, length, transhdrlen, flags, dontfrag);
}
1579
EXPORT_SYMBOL_GPL(ip6_append_data);
L
Linus Torvalds 已提交
1580

1581 1582
static void ip6_cork_release(struct inet_cork_full *cork,
			     struct inet6_cork *v6_cork)
1583
{
1584 1585 1586 1587 1588 1589 1590
	if (v6_cork->opt) {
		kfree(v6_cork->opt->dst0opt);
		kfree(v6_cork->opt->dst1opt);
		kfree(v6_cork->opt->hopopt);
		kfree(v6_cork->opt->srcrt);
		kfree(v6_cork->opt);
		v6_cork->opt = NULL;
1591 1592
	}

1593 1594 1595 1596
	if (cork->base.dst) {
		dst_release(cork->base.dst);
		cork->base.dst = NULL;
		cork->base.flags &= ~IPCORK_ALLFRAG;
1597
	}
1598
	memset(&cork->fl, 0, sizeof(cork->fl));
1599 1600
}

V
Vlad Yasevich 已提交
1601 1602 1603 1604
struct sk_buff *__ip6_make_skb(struct sock *sk,
			       struct sk_buff_head *queue,
			       struct inet_cork_full *cork,
			       struct inet6_cork *v6_cork)
L
Linus Torvalds 已提交
1605 1606 1607 1608 1609
{
	struct sk_buff *skb, *tmp_skb;
	struct sk_buff **tail_skb;
	struct in6_addr final_dst_buf, *final_dst = &final_dst_buf;
	struct ipv6_pinfo *np = inet6_sk(sk);
1610
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
1611
	struct ipv6hdr *hdr;
V
Vlad Yasevich 已提交
1612 1613 1614
	struct ipv6_txoptions *opt = v6_cork->opt;
	struct rt6_info *rt = (struct rt6_info *)cork->base.dst;
	struct flowi6 *fl6 = &cork->fl.u.ip6;
1615
	unsigned char proto = fl6->flowi6_proto;
L
Linus Torvalds 已提交
1616

V
Vlad Yasevich 已提交
1617
	skb = __skb_dequeue(queue);
1618
	if (!skb)
L
Linus Torvalds 已提交
1619 1620 1621 1622
		goto out;
	tail_skb = &(skb_shinfo(skb)->frag_list);

	/* move skb->data to ip header from ext header */
1623
	if (skb->data < skb_network_header(skb))
1624
		__skb_pull(skb, skb_network_offset(skb));
V
Vlad Yasevich 已提交
1625
	while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1626
		__skb_pull(tmp_skb, skb_network_header_len(skb));
L
Linus Torvalds 已提交
1627 1628 1629 1630 1631 1632 1633 1634 1635
		*tail_skb = tmp_skb;
		tail_skb = &(tmp_skb->next);
		skb->len += tmp_skb->len;
		skb->data_len += tmp_skb->len;
		skb->truesize += tmp_skb->truesize;
		tmp_skb->destructor = NULL;
		tmp_skb->sk = NULL;
	}

1636
	/* Allow local fragmentation. */
W
WANG Cong 已提交
1637
	skb->ignore_df = ip6_sk_ignore_df(sk);
1638

A
Alexey Dobriyan 已提交
1639
	*final_dst = fl6->daddr;
1640
	__skb_pull(skb, skb_network_header_len(skb));
L
Linus Torvalds 已提交
1641 1642 1643 1644 1645
	if (opt && opt->opt_flen)
		ipv6_push_frag_opts(skb, opt, &proto);
	if (opt && opt->opt_nflen)
		ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst);

1646 1647
	skb_push(skb, sizeof(struct ipv6hdr));
	skb_reset_network_header(skb);
1648
	hdr = ipv6_hdr(skb);
1649

V
Vlad Yasevich 已提交
1650
	ip6_flow_hdr(hdr, v6_cork->tclass,
1651
		     ip6_make_flowlabel(net, skb, fl6->flowlabel,
1652
					np->autoflowlabel, fl6));
V
Vlad Yasevich 已提交
1653
	hdr->hop_limit = v6_cork->hop_limit;
L
Linus Torvalds 已提交
1654
	hdr->nexthdr = proto;
A
Alexey Dobriyan 已提交
1655 1656
	hdr->saddr = fl6->saddr;
	hdr->daddr = *final_dst;
L
Linus Torvalds 已提交
1657

1658
	skb->priority = sk->sk_priority;
1659
	skb->mark = sk->sk_mark;
1660

1661
	skb_dst_set(skb, dst_clone(&rt->dst));
1662
	IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len);
1663
	if (proto == IPPROTO_ICMPV6) {
E
Eric Dumazet 已提交
1664
		struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
1665

1666 1667
		ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type);
		ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
1668 1669
	}

V
Vlad Yasevich 已提交
1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
	ip6_cork_release(cork, v6_cork);
out:
	return skb;
}

int ip6_send_skb(struct sk_buff *skb)
{
	struct net *net = sock_net(skb->sk);
	struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
	int err;

H
Herbert Xu 已提交
1681
	err = ip6_local_out(skb);
L
Linus Torvalds 已提交
1682 1683
	if (err) {
		if (err > 0)
E
Eric Dumazet 已提交
1684
			err = net_xmit_errno(err);
L
Linus Torvalds 已提交
1685
		if (err)
V
Vlad Yasevich 已提交
1686 1687
			IP6_INC_STATS(net, rt->rt6i_idev,
				      IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1688 1689 1690
	}

	return err;
V
Vlad Yasevich 已提交
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
}

int ip6_push_pending_frames(struct sock *sk)
{
	struct sk_buff *skb;

	skb = ip6_finish_skb(sk);
	if (!skb)
		return 0;

	return ip6_send_skb(skb);
L
Linus Torvalds 已提交
1702
}
1703
EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
L
Linus Torvalds 已提交
1704

1705
static void __ip6_flush_pending_frames(struct sock *sk,
V
Vlad Yasevich 已提交
1706 1707 1708
				       struct sk_buff_head *queue,
				       struct inet_cork_full *cork,
				       struct inet6_cork *v6_cork)
L
Linus Torvalds 已提交
1709 1710 1711
{
	struct sk_buff *skb;

1712
	while ((skb = __skb_dequeue_tail(queue)) != NULL) {
E
Eric Dumazet 已提交
1713 1714
		if (skb_dst(skb))
			IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
1715
				      IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1716 1717 1718
		kfree_skb(skb);
	}

V
Vlad Yasevich 已提交
1719
	ip6_cork_release(cork, v6_cork);
L
Linus Torvalds 已提交
1720
}
1721 1722 1723

void ip6_flush_pending_frames(struct sock *sk)
{
V
Vlad Yasevich 已提交
1724 1725
	__ip6_flush_pending_frames(sk, &sk->sk_write_queue,
				   &inet_sk(sk)->cork, &inet6_sk(sk)->cork);
1726
}
1727
EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
V
Vlad Yasevich 已提交
1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770

struct sk_buff *ip6_make_skb(struct sock *sk,
			     int getfrag(void *from, char *to, int offset,
					 int len, int odd, struct sk_buff *skb),
			     void *from, int length, int transhdrlen,
			     int hlimit, int tclass,
			     struct ipv6_txoptions *opt, struct flowi6 *fl6,
			     struct rt6_info *rt, unsigned int flags,
			     int dontfrag)
{
	struct inet_cork_full cork;
	struct inet6_cork v6_cork;
	struct sk_buff_head queue;
	int exthdrlen = (opt ? opt->opt_flen : 0);
	int err;

	if (flags & MSG_PROBE)
		return NULL;

	__skb_queue_head_init(&queue);

	cork.base.flags = 0;
	cork.base.addr = 0;
	cork.base.opt = NULL;
	v6_cork.opt = NULL;
	err = ip6_setup_cork(sk, &cork, &v6_cork, hlimit, tclass, opt, rt, fl6);
	if (err)
		return ERR_PTR(err);

	if (dontfrag < 0)
		dontfrag = inet6_sk(sk)->dontfrag;

	err = __ip6_append_data(sk, fl6, &queue, &cork.base, &v6_cork,
				&current->task_frag, getfrag, from,
				length + exthdrlen, transhdrlen + exthdrlen,
				flags, dontfrag);
	if (err) {
		__ip6_flush_pending_frames(sk, &queue, &cork, &v6_cork);
		return ERR_PTR(err);
	}

	return __ip6_make_skb(sk, &queue, &cork, &v6_cork);
}