ip_output.c 40.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		The Internet Protocol (IP) output module.
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Donald Becker, <becker@super.org>
 *		Alan Cox, <Alan.Cox@linux.org>
 *		Richard Underwood
 *		Stefan Becker, <stefanb@yello.ping.de>
 *		Jorge Cwik, <jorge@laser.satlink.net>
 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *		Hirokazu Takahashi, <taka@valinux.co.jp>
 *
 *	See ip_input.c for original log
 *
 *	Fixes:
 *		Alan Cox	:	Missing nonblock feature in ip_build_xmit.
 *		Mike Kilburn	:	htons() missing in ip_build_xmit.
23
 *		Bradford Johnson:	Fix faulty handling of some frames when
L
Linus Torvalds 已提交
24 25 26 27 28 29 30 31 32 33
 *					no route is found.
 *		Alexander Demenshin:	Missing sk/skb free in ip_queue_xmit
 *					(in case if packet not accepted by
 *					output firewall rules)
 *		Mike McLagan	:	Routing by source
 *		Alexey Kuznetsov:	use new route cache
 *		Andi Kleen:		Fix broken PMTU recovery and remove
 *					some redundant tests.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
 *		Andi Kleen	: 	Replace ip_reply with ip_send_reply.
34 35 36
 *		Andi Kleen	:	Split fast and slow ip_build_xmit path
 *					for decreased register pressure on x86
 *					and more readibility.
L
Linus Torvalds 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
 *		Marc Boucher	:	When call_out_firewall returns FW_QUEUE,
 *					silently drop skb instead of failing with -EPERM.
 *		Detlev Wengorz	:	Copy protocol for fragments.
 *		Hirokazu Takahashi:	HW checksumming for outgoing UDP
 *					datagrams.
 *		Hirokazu Takahashi:	sendfile() on UDP works now.
 */

#include <asm/uaccess.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/errno.h>
A
Al Viro 已提交
52
#include <linux/highmem.h>
53
#include <linux/slab.h>
L
Linus Torvalds 已提交
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68

#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/init.h>

#include <net/snmp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
69
#include <net/xfrm.h>
L
Linus Torvalds 已提交
70 71 72 73 74 75
#include <linux/skbuff.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/icmp.h>
#include <net/checksum.h>
#include <net/inetpeer.h>
76
#include <net/lwtunnel.h>
L
Linus Torvalds 已提交
77 78 79 80
#include <linux/igmp.h>
#include <linux/netfilter_ipv4.h>
#include <linux/netfilter_bridge.h>
#include <linux/netlink.h>
81
#include <linux/tcp.h>
L
Linus Torvalds 已提交
82

83 84 85 86
static int
ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
	    unsigned int mtu,
	    int (*output)(struct net *, struct sock *, struct sk_buff *));
87

L
Linus Torvalds 已提交
88
/* Generate a checksum for an outgoing IP datagram. */
89
void ip_send_check(struct iphdr *iph)
L
Linus Torvalds 已提交
90 91 92 93
{
	iph->check = 0;
	iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
}
E
Eric Dumazet 已提交
94
EXPORT_SYMBOL(ip_send_check);
L
Linus Torvalds 已提交
95

96
int __ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
H
Herbert Xu 已提交
97 98 99 100 101
{
	struct iphdr *iph = ip_hdr(skb);

	iph->tot_len = htons(skb->len);
	ip_send_check(iph);
102 103 104 105 106 107 108 109

	/* if egress device is enslaved to an L3 master device pass the
	 * skb to its handler for processing
	 */
	skb = l3mdev_ip_out(sk, skb);
	if (unlikely(!skb))
		return 0;

110 111
	return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT,
		       net, sk, skb, NULL, skb_dst(skb)->dev,
112
		       dst_output);
113 114
}

115
int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
H
Herbert Xu 已提交
116 117 118
{
	int err;

119
	err = __ip_local_out(net, sk, skb);
H
Herbert Xu 已提交
120
	if (likely(err == 1))
121
		err = dst_output(net, sk, skb);
H
Herbert Xu 已提交
122 123 124

	return err;
}
125
EXPORT_SYMBOL_GPL(ip_local_out);
H
Herbert Xu 已提交
126

L
Linus Torvalds 已提交
127 128 129 130 131
static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
{
	int ttl = inet->uc_ttl;

	if (ttl < 0)
132
		ttl = ip4_dst_hoplimit(dst);
L
Linus Torvalds 已提交
133 134 135
	return ttl;
}

136
/*
L
Linus Torvalds 已提交
137 138 139
 *		Add an ip header to a skbuff and send it out.
 *
 */
140
int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk,
141
			  __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)
L
Linus Torvalds 已提交
142 143
{
	struct inet_sock *inet = inet_sk(sk);
E
Eric Dumazet 已提交
144
	struct rtable *rt = skb_rtable(skb);
145
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
146 147 148
	struct iphdr *iph;

	/* Build the IP header. */
149
	skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
150
	skb_reset_network_header(skb);
151
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
152 153 154
	iph->version  = 4;
	iph->ihl      = 5;
	iph->tos      = inet->tos;
155
	iph->ttl      = ip_select_ttl(inet, &rt->dst);
156 157
	iph->daddr    = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
	iph->saddr    = saddr;
L
Linus Torvalds 已提交
158
	iph->protocol = sk->sk_protocol;
159 160 161 162 163
	if (ip_dont_fragment(sk, &rt->dst)) {
		iph->frag_off = htons(IP_DF);
		iph->id = 0;
	} else {
		iph->frag_off = 0;
164
		__ip_select_ident(net, iph, 1);
165
	}
L
Linus Torvalds 已提交
166

167 168 169
	if (opt && opt->opt.optlen) {
		iph->ihl += opt->opt.optlen>>2;
		ip_options_build(skb, &opt->opt, daddr, rt, 0);
L
Linus Torvalds 已提交
170 171 172
	}

	skb->priority = sk->sk_priority;
173
	skb->mark = sk->sk_mark;
L
Linus Torvalds 已提交
174 175

	/* Send it out. */
176
	return ip_local_out(net, skb->sk, skb);
L
Linus Torvalds 已提交
177
}
178 179
EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);

180
static int ip_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
181
{
E
Eric Dumazet 已提交
182
	struct dst_entry *dst = skb_dst(skb);
183
	struct rtable *rt = (struct rtable *)dst;
L
Linus Torvalds 已提交
184
	struct net_device *dev = dst->dev;
185
	unsigned int hh_len = LL_RESERVED_SPACE(dev);
186
	struct neighbour *neigh;
187
	u32 nexthop;
L
Linus Torvalds 已提交
188

189
	if (rt->rt_type == RTN_MULTICAST) {
190
		IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTMCAST, skb->len);
191
	} else if (rt->rt_type == RTN_BROADCAST)
192
		IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTBCAST, skb->len);
193

L
Linus Torvalds 已提交
194
	/* Be paranoid, rather than too clever. */
195
	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
L
Linus Torvalds 已提交
196 197 198
		struct sk_buff *skb2;

		skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
199
		if (!skb2) {
L
Linus Torvalds 已提交
200 201 202 203 204
			kfree_skb(skb);
			return -ENOMEM;
		}
		if (skb->sk)
			skb_set_owner_w(skb2, skb->sk);
205
		consume_skb(skb);
L
Linus Torvalds 已提交
206 207 208
		skb = skb2;
	}

209 210 211 212 213 214 215
	if (lwtunnel_xmit_redirect(dst->lwtstate)) {
		int res = lwtunnel_xmit(skb);

		if (res < 0 || res == LWTUNNEL_XMIT_DONE)
			return res;
	}

216
	rcu_read_lock_bh();
J
Julian Anastasov 已提交
217
	nexthop = (__force u32) rt_nexthop(rt, ip_hdr(skb)->daddr);
218 219 220
	neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
	if (unlikely(!neigh))
		neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
221
	if (!IS_ERR(neigh)) {
222
		int res = dst_neigh_output(dst, neigh, skb);
223

224
		rcu_read_unlock_bh();
225 226
		return res;
	}
227
	rcu_read_unlock_bh();
228

229 230
	net_dbg_ratelimited("%s: No header cache and no neighbour!\n",
			    __func__);
L
Linus Torvalds 已提交
231 232 233 234
	kfree_skb(skb);
	return -EINVAL;
}

235 236
static int ip_finish_output_gso(struct net *net, struct sock *sk,
				struct sk_buff *skb, unsigned int mtu)
237 238 239 240 241
{
	netdev_features_t features;
	struct sk_buff *segs;
	int ret = 0;

242
	/* common case: seglen is <= mtu
243
	 */
244
	if (skb_gso_validate_mtu(skb, mtu))
245
		return ip_finish_output2(net, sk, skb);
246

247
	/* Slowpath -  GSO segment length exceeds the egress MTU.
248
	 *
249 250 251 252 253 254 255 256 257 258
	 * This can happen in several cases:
	 *  - Forwarding of a TCP GRO skb, when DF flag is not set.
	 *  - Forwarding of an skb that arrived on a virtualization interface
	 *    (virtio-net/vhost/tap) with TSO/GSO size set by other network
	 *    stack.
	 *  - Local GSO skb transmitted on an NETIF_F_TSO tunnel stacked over an
	 *    interface with a smaller MTU.
	 *  - Arriving GRO skb (or GSO skb in a virtualized environment) that is
	 *    bridged to a NETIF_F_TSO tunnel stacked over an interface with an
	 *    insufficent MTU.
259 260
	 */
	features = netif_skb_features(skb);
261
	BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_SGO_CB_OFFSET);
262
	segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
263
	if (IS_ERR_OR_NULL(segs)) {
264 265 266 267 268 269 270 271 272 273 274
		kfree_skb(skb);
		return -ENOMEM;
	}

	consume_skb(skb);

	do {
		struct sk_buff *nskb = segs->next;
		int err;

		segs->next = NULL;
275
		err = ip_fragment(net, sk, segs, mtu, ip_finish_output2);
276 277 278 279 280 281 282 283 284

		if (err && ret == 0)
			ret = err;
		segs = nskb;
	} while (segs);

	return ret;
}

285
static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
286
{
287 288
	unsigned int mtu;

289 290
#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
	/* Policy lookup after SNAT yielded a new policy */
291
	if (skb_dst(skb)->xfrm) {
292
		IPCB(skb)->flags |= IPSKB_REROUTED;
293
		return dst_output(net, sk, skb);
294
	}
295
#endif
296
	mtu = ip_skb_dst_mtu(sk, skb);
297
	if (skb_is_gso(skb))
298
		return ip_finish_output_gso(net, sk, skb, mtu);
299

300
	if (skb->len > mtu || (IPCB(skb)->flags & IPSKB_FRAG_PMTU))
301
		return ip_fragment(net, sk, skb, mtu, ip_finish_output2);
302

303
	return ip_finish_output2(net, sk, skb);
L
Linus Torvalds 已提交
304 305
}

E
Eric W. Biederman 已提交
306
int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
307
{
E
Eric Dumazet 已提交
308
	struct rtable *rt = skb_rtable(skb);
309
	struct net_device *dev = rt->dst.dev;
L
Linus Torvalds 已提交
310 311 312 313

	/*
	 *	If the indicated interface is up and running, send the packet.
	 */
314
	IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
L
Linus Torvalds 已提交
315 316 317 318 319 320 321 322 323

	skb->dev = dev;
	skb->protocol = htons(ETH_P_IP);

	/*
	 *	Multicasts are looped back for other local users
	 */

	if (rt->rt_flags&RTCF_MULTICAST) {
324
		if (sk_mc_loop(sk)
L
Linus Torvalds 已提交
325 326 327 328 329 330 331 332 333
#ifdef CONFIG_IP_MROUTE
		/* Small optimization: do not loopback not local frames,
		   which returned after forwarding; they will be  dropped
		   by ip_mr_input in any case.
		   Note, that local frames are looped back to be delivered
		   to local recipients.

		   This check is duplicated in ip_mr_input at the moment.
		 */
334 335 336
		    &&
		    ((rt->rt_flags & RTCF_LOCAL) ||
		     !(IPCB(skb)->flags & IPSKB_FORWARDED))
L
Linus Torvalds 已提交
337
#endif
338
		   ) {
L
Linus Torvalds 已提交
339 340
			struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
			if (newskb)
341
				NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
342
					net, sk, newskb, NULL, newskb->dev,
343
					dev_loopback_xmit);
L
Linus Torvalds 已提交
344 345 346 347
		}

		/* Multicasts with ttl 0 must not go beyond the host */

348
		if (ip_hdr(skb)->ttl == 0) {
L
Linus Torvalds 已提交
349 350 351 352 353 354 355 356
			kfree_skb(skb);
			return 0;
		}
	}

	if (rt->rt_flags&RTCF_BROADCAST) {
		struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
		if (newskb)
357 358 359
			NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
				net, sk, newskb, NULL, newskb->dev,
				dev_loopback_xmit);
L
Linus Torvalds 已提交
360 361
	}

362 363 364
	return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
			    net, sk, skb, NULL, skb->dev,
			    ip_finish_output,
365
			    !(IPCB(skb)->flags & IPSKB_REROUTED));
L
Linus Torvalds 已提交
366 367
}

E
Eric W. Biederman 已提交
368
int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
369
{
E
Eric Dumazet 已提交
370
	struct net_device *dev = skb_dst(skb)->dev;
371

372
	IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
L
Linus Torvalds 已提交
373

374 375 376
	skb->dev = dev;
	skb->protocol = htons(ETH_P_IP);

377 378
	return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
			    net, sk, skb, NULL, dev,
379
			    ip_finish_output,
380
			    !(IPCB(skb)->flags & IPSKB_REROUTED));
L
Linus Torvalds 已提交
381 382
}

383 384 385 386 387 388 389 390 391 392 393 394 395 396
/*
 * copy saddr and daddr, possibly using 64bit load/stores
 * Equivalent to :
 *   iph->saddr = fl4->saddr;
 *   iph->daddr = fl4->daddr;
 */
static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4)
{
	BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) !=
		     offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr));
	memcpy(&iph->saddr, &fl4->saddr,
	       sizeof(fl4->saddr) + sizeof(fl4->daddr));
}

397 398
/* Note: skb->sk can be different from sk, in case of tunnels */
int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
L
Linus Torvalds 已提交
399 400
{
	struct inet_sock *inet = inet_sk(sk);
401
	struct net *net = sock_net(sk);
402
	struct ip_options_rcu *inet_opt;
403
	struct flowi4 *fl4;
L
Linus Torvalds 已提交
404 405
	struct rtable *rt;
	struct iphdr *iph;
406
	int res;
L
Linus Torvalds 已提交
407 408 409 410

	/* Skip all of this if the packet is already routed,
	 * f.e. by something like SCTP.
	 */
411
	rcu_read_lock();
412
	inet_opt = rcu_dereference(inet->inet_opt);
413
	fl4 = &fl->u.ip4;
E
Eric Dumazet 已提交
414
	rt = skb_rtable(skb);
415
	if (rt)
L
Linus Torvalds 已提交
416 417 418 419
		goto packet_routed;

	/* Make sure we can route this packet. */
	rt = (struct rtable *)__sk_dst_check(sk, 0);
420
	if (!rt) {
A
Al Viro 已提交
421
		__be32 daddr;
L
Linus Torvalds 已提交
422 423

		/* Use correct destination address if we have options. */
E
Eric Dumazet 已提交
424
		daddr = inet->inet_daddr;
425 426
		if (inet_opt && inet_opt->opt.srr)
			daddr = inet_opt->opt.faddr;
L
Linus Torvalds 已提交
427

428 429 430 431
		/* If this fails, retransmit mechanism of transport layer will
		 * keep trying until route appears or the connection times
		 * itself out.
		 */
432
		rt = ip_route_output_ports(net, fl4, sk,
433 434 435 436 437 438 439 440
					   daddr, inet->inet_saddr,
					   inet->inet_dport,
					   inet->inet_sport,
					   sk->sk_protocol,
					   RT_CONN_FLAGS(sk),
					   sk->sk_bound_dev_if);
		if (IS_ERR(rt))
			goto no_route;
441
		sk_setup_caps(sk, &rt->dst);
L
Linus Torvalds 已提交
442
	}
443
	skb_dst_set_noref(skb, &rt->dst);
L
Linus Torvalds 已提交
444 445

packet_routed:
J
Julian Anastasov 已提交
446
	if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway)
L
Linus Torvalds 已提交
447 448 449
		goto no_route;

	/* OK, we know where to send it, allocate and build IP header. */
450
	skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
451
	skb_reset_network_header(skb);
452
	iph = ip_hdr(skb);
453
	*((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
W
WANG Cong 已提交
454
	if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df)
L
Linus Torvalds 已提交
455 456 457
		iph->frag_off = htons(IP_DF);
	else
		iph->frag_off = 0;
458
	iph->ttl      = ip_select_ttl(inet, &rt->dst);
L
Linus Torvalds 已提交
459
	iph->protocol = sk->sk_protocol;
460 461
	ip_copy_addrs(iph, fl4);

L
Linus Torvalds 已提交
462 463
	/* Transport layer set skb->h.foo itself. */

464 465 466
	if (inet_opt && inet_opt->opt.optlen) {
		iph->ihl += inet_opt->opt.optlen >> 2;
		ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
L
Linus Torvalds 已提交
467 468
	}

469
	ip_select_ident_segs(net, skb, sk,
470
			     skb_shinfo(skb)->gso_segs ?: 1);
L
Linus Torvalds 已提交
471

472
	/* TODO : should we use skb->sk here instead of sk ? */
L
Linus Torvalds 已提交
473
	skb->priority = sk->sk_priority;
474
	skb->mark = sk->sk_mark;
L
Linus Torvalds 已提交
475

476
	res = ip_local_out(net, sk, skb);
477 478
	rcu_read_unlock();
	return res;
L
Linus Torvalds 已提交
479 480

no_route:
481
	rcu_read_unlock();
482
	IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
L
Linus Torvalds 已提交
483 484 485
	kfree_skb(skb);
	return -EHOSTUNREACH;
}
E
Eric Dumazet 已提交
486
EXPORT_SYMBOL(ip_queue_xmit);
L
Linus Torvalds 已提交
487 488 489 490 491 492

static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
{
	to->pkt_type = from->pkt_type;
	to->priority = from->priority;
	to->protocol = from->protocol;
E
Eric Dumazet 已提交
493
	skb_dst_drop(to);
494
	skb_dst_copy(to, from);
L
Linus Torvalds 已提交
495
	to->dev = from->dev;
T
Thomas Graf 已提交
496
	to->mark = from->mark;
L
Linus Torvalds 已提交
497 498 499 500 501 502 503

	/* Copy the flags to each fragment. */
	IPCB(to)->flags = IPCB(from)->flags;

#ifdef CONFIG_NET_SCHED
	to->tc_index = from->tc_index;
#endif
504
	nf_copy(to, from);
505
#if IS_ENABLED(CONFIG_IP_VS)
506
	to->ipvs_property = from->ipvs_property;
L
Linus Torvalds 已提交
507
#endif
508
	skb_copy_secmark(to, from);
L
Linus Torvalds 已提交
509 510
}

511
static int ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
512
		       unsigned int mtu,
513
		       int (*output)(struct net *, struct sock *, struct sk_buff *))
514 515 516
{
	struct iphdr *iph = ip_hdr(skb);

517
	if ((iph->frag_off & htons(IP_DF)) == 0)
518
		return ip_do_fragment(net, sk, skb, output);
519 520

	if (unlikely(!skb->ignore_df ||
521 522
		     (IPCB(skb)->frag_max_size &&
		      IPCB(skb)->frag_max_size > mtu))) {
523
		IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
524 525 526 527 528 529
		icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
			  htonl(mtu));
		kfree_skb(skb);
		return -EMSGSIZE;
	}

530
	return ip_do_fragment(net, sk, skb, output);
531 532
}

L
Linus Torvalds 已提交
533 534 535 536 537 538 539
/*
 *	This IP datagram is too large to be sent in one piece.  Break it up into
 *	smaller pieces (each of size equal to IP header plus
 *	a block of the data of the original IP data part) that will yet fit in a
 *	single device frame, and queue such a frame for sending.
 */

540 541
int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
		   int (*output)(struct net *, struct sock *, struct sk_buff *))
L
Linus Torvalds 已提交
542 543 544 545
{
	struct iphdr *iph;
	int ptr;
	struct sk_buff *skb2;
546
	unsigned int mtu, hlen, left, len, ll_rs;
L
Linus Torvalds 已提交
547
	int offset;
548
	__be16 not_last_frag;
E
Eric Dumazet 已提交
549
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
550 551
	int err = 0;

552 553 554 555 556
	/* for offloaded checksums cleanup checksum before fragmentation */
	if (skb->ip_summed == CHECKSUM_PARTIAL &&
	    (err = skb_checksum_help(skb)))
		goto fail;

L
Linus Torvalds 已提交
557 558 559 560
	/*
	 *	Point into the IP datagram header.
	 */

561
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
562

563
	mtu = ip_skb_dst_mtu(sk, skb);
564 565
	if (IPCB(skb)->frag_max_size && IPCB(skb)->frag_max_size < mtu)
		mtu = IPCB(skb)->frag_max_size;
L
Linus Torvalds 已提交
566 567 568 569 570 571

	/*
	 *	Setup starting values.
	 */

	hlen = iph->ihl * 4;
572
	mtu = mtu - hlen;	/* Size of data space */
H
Herbert Xu 已提交
573
	IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
L
Linus Torvalds 已提交
574 575 576 577 578 579 580 581

	/* When frag_list is given, use it. First, check its validity:
	 * some transformers could create wrong frag_list or break existing
	 * one, it is not prohibited. In this case fall back to copying.
	 *
	 * LATER: this step can be merged to real generation of fragments,
	 * we can switch to copy when see the first bad fragment.
	 */
582
	if (skb_has_frag_list(skb)) {
583
		struct sk_buff *frag, *frag2;
L
Linus Torvalds 已提交
584 585 586 587
		int first_len = skb_pagelen(skb);

		if (first_len - hlen > mtu ||
		    ((first_len - hlen) & 7) ||
588
		    ip_is_fragment(iph) ||
L
Linus Torvalds 已提交
589 590 591
		    skb_cloned(skb))
			goto slow_path;

592
		skb_walk_frags(skb, frag) {
L
Linus Torvalds 已提交
593 594 595 596
			/* Correct geometry. */
			if (frag->len > mtu ||
			    ((frag->len & 7) && frag->next) ||
			    skb_headroom(frag) < hlen)
597
				goto slow_path_clean;
L
Linus Torvalds 已提交
598 599 600

			/* Partially cloned skb? */
			if (skb_shared(frag))
601
				goto slow_path_clean;
602 603 604 605 606 607

			BUG_ON(frag->sk);
			if (skb->sk) {
				frag->sk = skb->sk;
				frag->destructor = sock_wfree;
			}
608
			skb->truesize -= frag->truesize;
L
Linus Torvalds 已提交
609 610 611 612 613 614 615
		}

		/* Everything is OK. Generate! */

		err = 0;
		offset = 0;
		frag = skb_shinfo(skb)->frag_list;
616
		skb_frag_list_init(skb);
L
Linus Torvalds 已提交
617 618 619 620 621 622 623 624 625 626 627
		skb->data_len = first_len - skb_headlen(skb);
		skb->len = first_len;
		iph->tot_len = htons(first_len);
		iph->frag_off = htons(IP_MF);
		ip_send_check(iph);

		for (;;) {
			/* Prepare header of the next frame,
			 * before previous one went down. */
			if (frag) {
				frag->ip_summed = CHECKSUM_NONE;
628
				skb_reset_transport_header(frag);
629 630
				__skb_push(frag, hlen);
				skb_reset_network_header(frag);
631
				memcpy(skb_network_header(frag), iph, hlen);
632
				iph = ip_hdr(frag);
L
Linus Torvalds 已提交
633 634 635 636 637 638
				iph->tot_len = htons(frag->len);
				ip_copy_metadata(frag, skb);
				if (offset == 0)
					ip_options_fragment(frag);
				offset += skb->len - hlen;
				iph->frag_off = htons(offset>>3);
639
				if (frag->next)
L
Linus Torvalds 已提交
640 641 642 643 644
					iph->frag_off |= htons(IP_MF);
				/* Ready, complete checksum */
				ip_send_check(iph);
			}

645
			err = output(net, sk, skb);
L
Linus Torvalds 已提交
646

647
			if (!err)
648
				IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
649 650 651 652 653 654 655 656 657
			if (err || !frag)
				break;

			skb = frag;
			frag = skb->next;
			skb->next = NULL;
		}

		if (err == 0) {
658
			IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
L
Linus Torvalds 已提交
659 660 661 662 663 664 665 666
			return 0;
		}

		while (frag) {
			skb = frag->next;
			kfree_skb(frag);
			frag = skb;
		}
667
		IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
668
		return err;
669 670 671 672 673 674 675 676 677

slow_path_clean:
		skb_walk_frags(skb, frag2) {
			if (frag2 == frag)
				break;
			frag2->sk = NULL;
			frag2->destructor = NULL;
			skb->truesize += frag2->truesize;
		}
L
Linus Torvalds 已提交
678 679 680
	}

slow_path:
681
	iph = ip_hdr(skb);
682

L
Linus Torvalds 已提交
683
	left = skb->len - hlen;		/* Space per frame */
684
	ptr = hlen;		/* Where to start from */
L
Linus Torvalds 已提交
685

686
	ll_rs = LL_RESERVED_SPACE(rt->dst.dev);
687

L
Linus Torvalds 已提交
688 689 690 691 692 693 694 695 696 697 698
	/*
	 *	Fragment the datagram.
	 */

	offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
	not_last_frag = iph->frag_off & htons(IP_MF);

	/*
	 *	Keep copying data until we run out.
	 */

S
Stephen Hemminger 已提交
699
	while (left > 0) {
L
Linus Torvalds 已提交
700 701 702 703
		len = left;
		/* IF: it doesn't fit, use 'mtu' - the data space left */
		if (len > mtu)
			len = mtu;
L
Lucas De Marchi 已提交
704
		/* IF: we are not sending up to and including the packet end
L
Linus Torvalds 已提交
705 706 707 708 709
		   then align the next start on an eight byte boundary */
		if (len < left)	{
			len &= ~7;
		}

710 711 712
		/* Allocate buffer */
		skb2 = alloc_skb(len + hlen + ll_rs, GFP_ATOMIC);
		if (!skb2) {
L
Linus Torvalds 已提交
713 714 715 716 717 718 719 720 721 722 723
			err = -ENOMEM;
			goto fail;
		}

		/*
		 *	Set up data on packet
		 */

		ip_copy_metadata(skb2, skb);
		skb_reserve(skb2, ll_rs);
		skb_put(skb2, len + hlen);
724
		skb_reset_network_header(skb2);
725
		skb2->transport_header = skb2->network_header + hlen;
L
Linus Torvalds 已提交
726 727 728 729 730 731 732 733 734 735 736 737 738

		/*
		 *	Charge the memory for the fragment to any owner
		 *	it might possess
		 */

		if (skb->sk)
			skb_set_owner_w(skb2, skb->sk);

		/*
		 *	Copy the packet header into the new buffer.
		 */

739
		skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
L
Linus Torvalds 已提交
740 741 742 743

		/*
		 *	Copy a block of the IP datagram.
		 */
744
		if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
L
Linus Torvalds 已提交
745 746 747 748 749 750
			BUG();
		left -= len;

		/*
		 *	Fill in the new header fields.
		 */
751
		iph = ip_hdr(skb2);
L
Linus Torvalds 已提交
752 753
		iph->frag_off = htons((offset >> 3));

754 755 756
		if (IPCB(skb)->flags & IPSKB_FRAG_PMTU)
			iph->frag_off |= htons(IP_DF);

L
Linus Torvalds 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
		/* ANK: dirty, but effective trick. Upgrade options only if
		 * the segment to be fragmented was THE FIRST (otherwise,
		 * options are already fixed) and make it ONCE
		 * on the initial skb, so that all the following fragments
		 * will inherit fixed options.
		 */
		if (offset == 0)
			ip_options_fragment(skb);

		/*
		 *	Added AC : If we are fragmenting a fragment that's not the
		 *		   last fragment then keep MF on each bit
		 */
		if (left > 0 || not_last_frag)
			iph->frag_off |= htons(IP_MF);
		ptr += len;
		offset += len;

		/*
		 *	Put this fragment into the sending queue.
		 */
		iph->tot_len = htons(len + hlen);

		ip_send_check(iph);

782
		err = output(net, sk, skb2);
L
Linus Torvalds 已提交
783 784
		if (err)
			goto fail;
785

786
		IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
787
	}
788
	consume_skb(skb);
789
	IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
L
Linus Torvalds 已提交
790 791 792
	return err;

fail:
793
	kfree_skb(skb);
794
	IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
795 796
	return err;
}
797
EXPORT_SYMBOL(ip_do_fragment);
798

L
Linus Torvalds 已提交
799 800 801
int
ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
{
802
	struct msghdr *msg = from;
L
Linus Torvalds 已提交
803

804
	if (skb->ip_summed == CHECKSUM_PARTIAL) {
805
		if (copy_from_iter(to, len, &msg->msg_iter) != len)
L
Linus Torvalds 已提交
806 807
			return -EFAULT;
	} else {
808
		__wsum csum = 0;
809
		if (csum_and_copy_from_iter(to, len, &csum, &msg->msg_iter) != len)
L
Linus Torvalds 已提交
810 811 812 813 814
			return -EFAULT;
		skb->csum = csum_block_add(skb->csum, csum, odd);
	}
	return 0;
}
E
Eric Dumazet 已提交
815
EXPORT_SYMBOL(ip_generic_getfrag);
L
Linus Torvalds 已提交
816

817
static inline __wsum
L
Linus Torvalds 已提交
818 819 820
csum_page(struct page *page, int offset, int copy)
{
	char *kaddr;
821
	__wsum csum;
L
Linus Torvalds 已提交
822 823 824 825 826 827
	kaddr = kmap(page);
	csum = csum_partial(kaddr + offset, copy, 0);
	kunmap(page);
	return csum;
}

A
Adrian Bunk 已提交
828
static inline int ip_ufo_append_data(struct sock *sk,
829
			struct sk_buff_head *queue,
830 831 832
			int getfrag(void *from, char *to, int offset, int len,
			       int odd, struct sk_buff *skb),
			void *from, int length, int hh_len, int fragheaderlen,
833
			int transhdrlen, int maxfraglen, unsigned int flags)
834 835 836 837 838 839 840 841
{
	struct sk_buff *skb;
	int err;

	/* There is support for UDP fragmentation offload by network
	 * device, so create one single skb packet containing complete
	 * udp datagram
	 */
842 843
	skb = skb_peek_tail(queue);
	if (!skb) {
844 845 846 847
		skb = sock_alloc_send_skb(sk,
			hh_len + fragheaderlen + transhdrlen + 20,
			(flags & MSG_DONTWAIT), &err);

848
		if (!skb)
849 850 851 852 853 854
			return err;

		/* reserve space for Hardware header */
		skb_reserve(skb, hh_len);

		/* create space for UDP/IP header */
855
		skb_put(skb, fragheaderlen + transhdrlen);
856 857

		/* initialize network header pointer */
858
		skb_reset_network_header(skb);
859 860

		/* initialize protocol header pointer */
861
		skb->transport_header = skb->network_header + fragheaderlen;
862 863 864

		skb->csum = 0;

865
		__skb_queue_tail(queue, skb);
866 867
	} else if (skb_is_gso(skb)) {
		goto append;
868
	}
869

870 871 872 873 874 875
	skb->ip_summed = CHECKSUM_PARTIAL;
	/* specify the length of each IP datagram fragment */
	skb_shinfo(skb)->gso_size = maxfraglen - fragheaderlen;
	skb_shinfo(skb)->gso_type = SKB_GSO_UDP;

append:
876 877
	return skb_append_datato_frags(sk, skb, getfrag, from,
				       (length - transhdrlen));
878 879
}

880 881 882
static int __ip_append_data(struct sock *sk,
			    struct flowi4 *fl4,
			    struct sk_buff_head *queue,
883
			    struct inet_cork *cork,
884
			    struct page_frag *pfrag,
885 886 887 888
			    int getfrag(void *from, char *to, int offset,
					int len, int odd, struct sk_buff *skb),
			    void *from, int length, int transhdrlen,
			    unsigned int flags)
L
Linus Torvalds 已提交
889 890 891 892
{
	struct inet_sock *inet = inet_sk(sk);
	struct sk_buff *skb;

893
	struct ip_options *opt = cork->opt;
L
Linus Torvalds 已提交
894 895 896 897 898 899
	int hh_len;
	int exthdrlen;
	int mtu;
	int copy;
	int err;
	int offset = 0;
900
	unsigned int maxfraglen, fragheaderlen, maxnonfragsize;
L
Linus Torvalds 已提交
901
	int csummode = CHECKSUM_NONE;
902
	struct rtable *rt = (struct rtable *)cork->dst;
903
	u32 tskey = 0;
L
Linus Torvalds 已提交
904

905 906 907
	skb = skb_peek_tail(queue);

	exthdrlen = !skb ? rt->dst.header_len : 0;
908
	mtu = cork->fragsize;
909 910 911
	if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&
	    sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
		tskey = sk->sk_tskey++;
L
Linus Torvalds 已提交
912

913
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
L
Linus Torvalds 已提交
914 915 916

	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
W
WANG Cong 已提交
917
	maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu;
L
Linus Torvalds 已提交
918

919
	if (cork->length + length > maxnonfragsize - fragheaderlen) {
920
		ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
921
			       mtu - (opt ? opt->optlen : 0));
L
Linus Torvalds 已提交
922 923 924 925 926 927 928 929 930
		return -EMSGSIZE;
	}

	/*
	 * transhdrlen > 0 means that this is the first fragment and we wish
	 * it won't be fragmented in the future.
	 */
	if (transhdrlen &&
	    length + fragheaderlen <= mtu &&
931
	    rt->dst.dev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM) &&
932
	    !(flags & MSG_MORE) &&
L
Linus Torvalds 已提交
933
	    !exthdrlen)
934
		csummode = CHECKSUM_PARTIAL;
L
Linus Torvalds 已提交
935

936
	cork->length += length;
937
	if (((length > mtu) || (skb && skb_is_gso(skb))) &&
938
	    (sk->sk_protocol == IPPROTO_UDP) &&
939
	    (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len &&
940
	    (sk->sk_type == SOCK_DGRAM) && !sk->sk_no_check_tx) {
941 942
		err = ip_ufo_append_data(sk, queue, getfrag, from, length,
					 hh_len, fragheaderlen, transhdrlen,
943
					 maxfraglen, flags);
944
		if (err)
945 946 947
			goto error;
		return 0;
	}
L
Linus Torvalds 已提交
948 949 950 951 952 953 954 955

	/* So, what's going on in the loop below?
	 *
	 * We use calculated fragment length to generate chained skb,
	 * each of segments is IP fragment ready for sending to network after
	 * adding appropriate IP header.
	 */

956
	if (!skb)
L
Linus Torvalds 已提交
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
		goto alloc_new_skb;

	while (length > 0) {
		/* Check if the remaining data fits into current packet. */
		copy = mtu - skb->len;
		if (copy < length)
			copy = maxfraglen - skb->len;
		if (copy <= 0) {
			char *data;
			unsigned int datalen;
			unsigned int fraglen;
			unsigned int fraggap;
			unsigned int alloclen;
			struct sk_buff *skb_prev;
alloc_new_skb:
			skb_prev = skb;
			if (skb_prev)
				fraggap = skb_prev->len - maxfraglen;
			else
				fraggap = 0;

			/*
			 * If remaining data exceeds the mtu,
			 * we know we need more fragment(s).
			 */
			datalen = length + fraggap;
			if (datalen > mtu - fragheaderlen)
				datalen = maxfraglen - fragheaderlen;
			fraglen = datalen + fragheaderlen;

987
			if ((flags & MSG_MORE) &&
988
			    !(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
989 990
				alloclen = mtu;
			else
991
				alloclen = fraglen;
L
Linus Torvalds 已提交
992

993 994
			alloclen += exthdrlen;

L
Linus Torvalds 已提交
995 996 997 998 999
			/* The last fragment gets additional space at tail.
			 * Note, with MSG_MORE we overallocate on fragments,
			 * because we have no idea what fragment will be
			 * the last.
			 */
1000
			if (datalen == length + fraggap)
1001
				alloclen += rt->dst.trailer_len;
1002

L
Linus Torvalds 已提交
1003
			if (transhdrlen) {
1004
				skb = sock_alloc_send_skb(sk,
L
Linus Torvalds 已提交
1005 1006 1007 1008 1009 1010
						alloclen + hh_len + 15,
						(flags & MSG_DONTWAIT), &err);
			} else {
				skb = NULL;
				if (atomic_read(&sk->sk_wmem_alloc) <=
				    2 * sk->sk_sndbuf)
1011
					skb = sock_wmalloc(sk,
L
Linus Torvalds 已提交
1012 1013
							   alloclen + hh_len + 15, 1,
							   sk->sk_allocation);
1014
				if (unlikely(!skb))
L
Linus Torvalds 已提交
1015 1016
					err = -ENOBUFS;
			}
1017
			if (!skb)
L
Linus Torvalds 已提交
1018 1019 1020 1021 1022 1023 1024 1025
				goto error;

			/*
			 *	Fill in the control structures
			 */
			skb->ip_summed = csummode;
			skb->csum = 0;
			skb_reserve(skb, hh_len);
1026 1027

			/* only the initial fragment is time stamped */
1028
			skb_shinfo(skb)->tx_flags = cork->tx_flags;
1029
			cork->tx_flags = 0;
1030 1031
			skb_shinfo(skb)->tskey = tskey;
			tskey = 0;
L
Linus Torvalds 已提交
1032 1033 1034 1035

			/*
			 *	Find where to start putting bytes.
			 */
1036
			data = skb_put(skb, fraglen + exthdrlen);
1037
			skb_set_network_header(skb, exthdrlen);
1038 1039
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
1040
			data += fragheaderlen + exthdrlen;
L
Linus Torvalds 已提交
1041 1042 1043 1044 1045 1046 1047 1048

			if (fraggap) {
				skb->csum = skb_copy_and_csum_bits(
					skb_prev, maxfraglen,
					data + transhdrlen, fraggap, 0);
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
				data += fraggap;
1049
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
			}

			copy = datalen - transhdrlen - fraggap;
			if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
				err = -EFAULT;
				kfree_skb(skb);
				goto error;
			}

			offset += copy;
			length -= datalen - fraggap;
			transhdrlen = 0;
			exthdrlen = 0;
			csummode = CHECKSUM_NONE;

			/*
			 * Put the packet on the pending queue.
			 */
1068
			__skb_queue_tail(queue, skb);
L
Linus Torvalds 已提交
1069 1070 1071 1072 1073 1074
			continue;
		}

		if (copy > length)
			copy = length;

1075
		if (!(rt->dst.dev->features&NETIF_F_SG)) {
L
Linus Torvalds 已提交
1076 1077 1078
			unsigned int off;

			off = skb->len;
1079
			if (getfrag(from, skb_put(skb, copy),
L
Linus Torvalds 已提交
1080 1081 1082 1083 1084 1085 1086 1087
					offset, copy, off, skb) < 0) {
				__skb_trim(skb, off);
				err = -EFAULT;
				goto error;
			}
		} else {
			int i = skb_shinfo(skb)->nr_frags;

1088 1089
			err = -ENOMEM;
			if (!sk_page_frag_refill(sk, pfrag))
L
Linus Torvalds 已提交
1090
				goto error;
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101

			if (!skb_can_coalesce(skb, i, pfrag->page,
					      pfrag->offset)) {
				err = -EMSGSIZE;
				if (i == MAX_SKB_FRAGS)
					goto error;

				__skb_fill_page_desc(skb, i, pfrag->page,
						     pfrag->offset, 0);
				skb_shinfo(skb)->nr_frags = ++i;
				get_page(pfrag->page);
L
Linus Torvalds 已提交
1102
			}
1103 1104 1105 1106 1107 1108 1109 1110
			copy = min_t(int, copy, pfrag->size - pfrag->offset);
			if (getfrag(from,
				    page_address(pfrag->page) + pfrag->offset,
				    offset, copy, skb->len, skb) < 0)
				goto error_efault;

			pfrag->offset += copy;
			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
L
Linus Torvalds 已提交
1111 1112
			skb->len += copy;
			skb->data_len += copy;
1113 1114
			skb->truesize += copy;
			atomic_add(copy, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1115 1116 1117 1118 1119 1120 1121
		}
		offset += copy;
		length -= copy;
	}

	return 0;

1122 1123
error_efault:
	err = -EFAULT;
L
Linus Torvalds 已提交
1124
error:
1125
	cork->length -= length;
P
Pavel Emelyanov 已提交
1126
	IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
1127
	return err;
L
Linus Torvalds 已提交
1128 1129
}

1130 1131 1132
static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
			 struct ipcm_cookie *ipc, struct rtable **rtp)
{
1133
	struct ip_options_rcu *opt;
1134 1135 1136 1137 1138 1139 1140
	struct rtable *rt;

	/*
	 * setup for corking.
	 */
	opt = ipc->opt;
	if (opt) {
1141
		if (!cork->opt) {
1142 1143
			cork->opt = kmalloc(sizeof(struct ip_options) + 40,
					    sk->sk_allocation);
1144
			if (unlikely(!cork->opt))
1145 1146
				return -ENOBUFS;
		}
1147
		memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen);
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
		cork->flags |= IPCORK_OPT;
		cork->addr = ipc->addr;
	}
	rt = *rtp;
	if (unlikely(!rt))
		return -EFAULT;
	/*
	 * We steal reference to this route, caller should not release it
	 */
	*rtp = NULL;
1158 1159
	cork->fragsize = ip_sk_use_pmtu(sk) ?
			 dst_mtu(&rt->dst) : rt->dst.dev->mtu;
1160 1161
	cork->dst = &rt->dst;
	cork->length = 0;
1162 1163 1164
	cork->ttl = ipc->ttl;
	cork->tos = ipc->tos;
	cork->priority = ipc->priority;
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	cork->tx_flags = ipc->tx_flags;

	return 0;
}

/*
 *	ip_append_data() and ip_append_page() can make one large IP datagram
 *	from many pieces of data. Each pieces will be holded on the socket
 *	until ip_push_pending_frames() is called. Each piece can be a page
 *	or non-page data.
 *
 *	Not only UDP, other transport protocols - e.g. raw sockets - can use
 *	this interface potentially.
 *
 *	LATER: length must be adjusted by pad at tail, when it is required.
 */
1181
int ip_append_data(struct sock *sk, struct flowi4 *fl4,
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
		   int getfrag(void *from, char *to, int offset, int len,
			       int odd, struct sk_buff *skb),
		   void *from, int length, int transhdrlen,
		   struct ipcm_cookie *ipc, struct rtable **rtp,
		   unsigned int flags)
{
	struct inet_sock *inet = inet_sk(sk);
	int err;

	if (flags&MSG_PROBE)
		return 0;

	if (skb_queue_empty(&sk->sk_write_queue)) {
1195
		err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp);
1196 1197 1198 1199 1200 1201
		if (err)
			return err;
	} else {
		transhdrlen = 0;
	}

1202 1203
	return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base,
				sk_page_frag(sk), getfrag,
1204 1205 1206
				from, length, transhdrlen, flags);
}

1207
ssize_t	ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,
L
Linus Torvalds 已提交
1208 1209 1210 1211 1212 1213
		       int offset, size_t size, int flags)
{
	struct inet_sock *inet = inet_sk(sk);
	struct sk_buff *skb;
	struct rtable *rt;
	struct ip_options *opt = NULL;
1214
	struct inet_cork *cork;
L
Linus Torvalds 已提交
1215 1216 1217 1218
	int hh_len;
	int mtu;
	int len;
	int err;
1219
	unsigned int maxfraglen, fragheaderlen, fraggap, maxnonfragsize;
L
Linus Torvalds 已提交
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229

	if (inet->hdrincl)
		return -EPERM;

	if (flags&MSG_PROBE)
		return 0;

	if (skb_queue_empty(&sk->sk_write_queue))
		return -EINVAL;

1230 1231 1232 1233
	cork = &inet->cork.base;
	rt = (struct rtable *)cork->dst;
	if (cork->flags & IPCORK_OPT)
		opt = cork->opt;
L
Linus Torvalds 已提交
1234

1235
	if (!(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
1236 1237
		return -EOPNOTSUPP;

1238
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1239
	mtu = cork->fragsize;
L
Linus Torvalds 已提交
1240 1241 1242

	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
W
WANG Cong 已提交
1243
	maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu;
L
Linus Torvalds 已提交
1244

1245
	if (cork->length + size > maxnonfragsize - fragheaderlen) {
1246 1247
		ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
			       mtu - (opt ? opt->optlen : 0));
L
Linus Torvalds 已提交
1248 1249 1250
		return -EMSGSIZE;
	}

1251 1252
	skb = skb_peek_tail(&sk->sk_write_queue);
	if (!skb)
L
Linus Torvalds 已提交
1253 1254
		return -EINVAL;

1255 1256
	if ((size + skb->len > mtu) &&
	    (sk->sk_protocol == IPPROTO_UDP) &&
1257
	    (rt->dst.dev->features & NETIF_F_UFO)) {
1258 1259 1260
		if (skb->ip_summed != CHECKSUM_PARTIAL)
			return -EOPNOTSUPP;

1261
		skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
H
Herbert Xu 已提交
1262
		skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1263
	}
1264
	cork->length += size;
1265

L
Linus Torvalds 已提交
1266
	while (size > 0) {
1267
		if (skb_is_gso(skb)) {
1268
			len = size;
1269
		} else {
1270 1271 1272 1273 1274 1275

			/* Check if the remaining data fits into current packet. */
			len = mtu - skb->len;
			if (len < size)
				len = maxfraglen - skb->len;
		}
L
Linus Torvalds 已提交
1276 1277 1278 1279 1280
		if (len <= 0) {
			struct sk_buff *skb_prev;
			int alloclen;

			skb_prev = skb;
1281
			fraggap = skb_prev->len - maxfraglen;
L
Linus Torvalds 已提交
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299

			alloclen = fragheaderlen + hh_len + fraggap + 15;
			skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
			if (unlikely(!skb)) {
				err = -ENOBUFS;
				goto error;
			}

			/*
			 *	Fill in the control structures
			 */
			skb->ip_summed = CHECKSUM_NONE;
			skb->csum = 0;
			skb_reserve(skb, hh_len);

			/*
			 *	Find where to start putting bytes.
			 */
1300
			skb_put(skb, fragheaderlen + fraggap);
1301
			skb_reset_network_header(skb);
1302 1303
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
L
Linus Torvalds 已提交
1304
			if (fraggap) {
1305 1306
				skb->csum = skb_copy_and_csum_bits(skb_prev,
								   maxfraglen,
1307
						    skb_transport_header(skb),
1308
								   fraggap, 0);
L
Linus Torvalds 已提交
1309 1310
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
1311
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
			}

			/*
			 * Put the packet on the pending queue.
			 */
			__skb_queue_tail(&sk->sk_write_queue, skb);
			continue;
		}

		if (len > size)
			len = size;
1323 1324

		if (skb_append_pagefrags(skb, page, offset, len)) {
L
Linus Torvalds 已提交
1325 1326 1327 1328 1329
			err = -EMSGSIZE;
			goto error;
		}

		if (skb->ip_summed == CHECKSUM_NONE) {
1330
			__wsum csum;
L
Linus Torvalds 已提交
1331 1332 1333 1334 1335 1336
			csum = csum_page(page, offset, len);
			skb->csum = csum_block_add(skb->csum, csum, skb->len);
		}

		skb->len += len;
		skb->data_len += len;
1337 1338
		skb->truesize += len;
		atomic_add(len, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1339 1340 1341 1342 1343 1344
		offset += len;
		size -= len;
	}
	return 0;

error:
1345
	cork->length -= size;
P
Pavel Emelyanov 已提交
1346
	IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1347 1348 1349
	return err;
}

1350
static void ip_cork_release(struct inet_cork *cork)
1351
{
1352 1353 1354 1355 1356
	cork->flags &= ~IPCORK_OPT;
	kfree(cork->opt);
	cork->opt = NULL;
	dst_release(cork->dst);
	cork->dst = NULL;
1357 1358
}

L
Linus Torvalds 已提交
1359 1360 1361 1362
/*
 *	Combined all pending IP fragments on the socket as one IP datagram
 *	and push them out.
 */
1363
struct sk_buff *__ip_make_skb(struct sock *sk,
1364
			      struct flowi4 *fl4,
1365 1366
			      struct sk_buff_head *queue,
			      struct inet_cork *cork)
L
Linus Torvalds 已提交
1367 1368 1369 1370
{
	struct sk_buff *skb, *tmp_skb;
	struct sk_buff **tail_skb;
	struct inet_sock *inet = inet_sk(sk);
1371
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
1372
	struct ip_options *opt = NULL;
1373
	struct rtable *rt = (struct rtable *)cork->dst;
L
Linus Torvalds 已提交
1374
	struct iphdr *iph;
1375
	__be16 df = 0;
L
Linus Torvalds 已提交
1376 1377
	__u8 ttl;

1378 1379
	skb = __skb_dequeue(queue);
	if (!skb)
L
Linus Torvalds 已提交
1380 1381 1382 1383
		goto out;
	tail_skb = &(skb_shinfo(skb)->frag_list);

	/* move skb->data to ip header from ext header */
1384
	if (skb->data < skb_network_header(skb))
1385
		__skb_pull(skb, skb_network_offset(skb));
1386
	while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1387
		__skb_pull(tmp_skb, skb_network_header_len(skb));
L
Linus Torvalds 已提交
1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
		*tail_skb = tmp_skb;
		tail_skb = &(tmp_skb->next);
		skb->len += tmp_skb->len;
		skb->data_len += tmp_skb->len;
		skb->truesize += tmp_skb->truesize;
		tmp_skb->destructor = NULL;
		tmp_skb->sk = NULL;
	}

	/* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
	 * to fragment the frame generated here. No matter, what transforms
	 * how transforms change size of the packet, it will come out.
	 */
W
WANG Cong 已提交
1401
	skb->ignore_df = ip_sk_ignore_df(sk);
L
Linus Torvalds 已提交
1402 1403

	/* DF bit is set when we want to see DF on outgoing frames.
W
WANG Cong 已提交
1404
	 * If ignore_df is set too, we still allow to fragment this frame
L
Linus Torvalds 已提交
1405
	 * locally. */
1406 1407
	if (inet->pmtudisc == IP_PMTUDISC_DO ||
	    inet->pmtudisc == IP_PMTUDISC_PROBE ||
1408 1409
	    (skb->len <= dst_mtu(&rt->dst) &&
	     ip_dont_fragment(sk, &rt->dst)))
L
Linus Torvalds 已提交
1410 1411
		df = htons(IP_DF);

1412 1413
	if (cork->flags & IPCORK_OPT)
		opt = cork->opt;
L
Linus Torvalds 已提交
1414

1415 1416 1417
	if (cork->ttl != 0)
		ttl = cork->ttl;
	else if (rt->rt_type == RTN_MULTICAST)
L
Linus Torvalds 已提交
1418 1419
		ttl = inet->mc_ttl;
	else
1420
		ttl = ip_select_ttl(inet, &rt->dst);
L
Linus Torvalds 已提交
1421

1422
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
1423 1424
	iph->version = 4;
	iph->ihl = 5;
1425
	iph->tos = (cork->tos != -1) ? cork->tos : inet->tos;
L
Linus Torvalds 已提交
1426 1427 1428
	iph->frag_off = df;
	iph->ttl = ttl;
	iph->protocol = sk->sk_protocol;
1429
	ip_copy_addrs(iph, fl4);
1430
	ip_select_ident(net, skb, sk);
L
Linus Torvalds 已提交
1431

1432 1433 1434 1435 1436
	if (opt) {
		iph->ihl += opt->optlen>>2;
		ip_options_build(skb, opt, cork->addr, rt, 0);
	}

1437
	skb->priority = (cork->tos != -1) ? cork->priority: sk->sk_priority;
1438
	skb->mark = sk->sk_mark;
1439 1440 1441 1442
	/*
	 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
	 * on dst refcount
	 */
1443
	cork->dst = NULL;
1444
	skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
1445

1446
	if (iph->protocol == IPPROTO_ICMP)
1447
		icmp_out_count(net, ((struct icmphdr *)
1448 1449
			skb_transport_header(skb))->type);

1450 1451 1452 1453 1454
	ip_cork_release(cork);
out:
	return skb;
}

E
Eric Dumazet 已提交
1455
int ip_send_skb(struct net *net, struct sk_buff *skb)
1456 1457 1458
{
	int err;

1459
	err = ip_local_out(net, skb->sk, skb);
L
Linus Torvalds 已提交
1460 1461
	if (err) {
		if (err > 0)
E
Eric Dumazet 已提交
1462
			err = net_xmit_errno(err);
L
Linus Torvalds 已提交
1463
		if (err)
1464
			IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1465 1466 1467 1468 1469
	}

	return err;
}

1470
int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4)
1471
{
1472 1473
	struct sk_buff *skb;

1474
	skb = ip_finish_skb(sk, fl4);
1475 1476 1477 1478
	if (!skb)
		return 0;

	/* Netfilter gets whole the not fragmented skb. */
E
Eric Dumazet 已提交
1479
	return ip_send_skb(sock_net(sk), skb);
1480 1481
}

L
Linus Torvalds 已提交
1482 1483 1484
/*
 *	Throw away all pending data on the socket.
 */
1485 1486 1487
static void __ip_flush_pending_frames(struct sock *sk,
				      struct sk_buff_head *queue,
				      struct inet_cork *cork)
L
Linus Torvalds 已提交
1488 1489 1490
{
	struct sk_buff *skb;

1491
	while ((skb = __skb_dequeue_tail(queue)) != NULL)
L
Linus Torvalds 已提交
1492 1493
		kfree_skb(skb);

1494 1495 1496 1497 1498
	ip_cork_release(cork);
}

void ip_flush_pending_frames(struct sock *sk)
{
1499
	__ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
L
Linus Torvalds 已提交
1500 1501
}

1502
struct sk_buff *ip_make_skb(struct sock *sk,
1503
			    struct flowi4 *fl4,
1504 1505 1506 1507 1508 1509
			    int getfrag(void *from, char *to, int offset,
					int len, int odd, struct sk_buff *skb),
			    void *from, int length, int transhdrlen,
			    struct ipcm_cookie *ipc, struct rtable **rtp,
			    unsigned int flags)
{
1510
	struct inet_cork cork;
1511 1512 1513 1514 1515 1516 1517 1518
	struct sk_buff_head queue;
	int err;

	if (flags & MSG_PROBE)
		return NULL;

	__skb_queue_head_init(&queue);

1519 1520
	cork.flags = 0;
	cork.addr = 0;
1521
	cork.opt = NULL;
1522 1523 1524 1525
	err = ip_setup_cork(sk, &cork, ipc, rtp);
	if (err)
		return ERR_PTR(err);

1526 1527
	err = __ip_append_data(sk, fl4, &queue, &cork,
			       &current->task_frag, getfrag,
1528 1529 1530 1531 1532 1533
			       from, length, transhdrlen, flags);
	if (err) {
		__ip_flush_pending_frames(sk, &queue, &cork);
		return ERR_PTR(err);
	}

1534
	return __ip_make_skb(sk, fl4, &queue, &cork);
1535
}
L
Linus Torvalds 已提交
1536 1537 1538 1539

/*
 *	Fetch data from kernel space and fill in checksum if needed.
 */
1540
static int ip_reply_glue_bits(void *dptr, char *to, int offset,
L
Linus Torvalds 已提交
1541 1542
			      int len, int odd, struct sk_buff *skb)
{
1543
	__wsum csum;
L
Linus Torvalds 已提交
1544 1545 1546

	csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
	skb->csum = csum_block_add(skb->csum, csum, odd);
1547
	return 0;
L
Linus Torvalds 已提交
1548 1549
}

1550
/*
L
Linus Torvalds 已提交
1551
 *	Generic function to send a packet as reply to another packet.
1552
 *	Used to send some TCP resets/acks so far.
L
Linus Torvalds 已提交
1553
 */
1554
void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb,
1555 1556 1557
			   const struct ip_options *sopt,
			   __be32 daddr, __be32 saddr,
			   const struct ip_reply_arg *arg,
1558
			   unsigned int len)
L
Linus Torvalds 已提交
1559
{
1560
	struct ip_options_data replyopts;
L
Linus Torvalds 已提交
1561
	struct ipcm_cookie ipc;
1562
	struct flowi4 fl4;
E
Eric Dumazet 已提交
1563
	struct rtable *rt = skb_rtable(skb);
1564
	struct net *net = sock_net(sk);
1565
	struct sk_buff *nskb;
1566
	int err;
1567
	int oif;
L
Linus Torvalds 已提交
1568

1569
	if (__ip_options_echo(&replyopts.opt.opt, skb, sopt))
L
Linus Torvalds 已提交
1570 1571
		return;

1572
	ipc.addr = daddr;
L
Linus Torvalds 已提交
1573
	ipc.opt = NULL;
1574
	ipc.tx_flags = 0;
1575 1576
	ipc.ttl = 0;
	ipc.tos = -1;
L
Linus Torvalds 已提交
1577

1578
	if (replyopts.opt.opt.optlen) {
L
Linus Torvalds 已提交
1579 1580
		ipc.opt = &replyopts.opt;

1581 1582
		if (replyopts.opt.opt.srr)
			daddr = replyopts.opt.opt.faddr;
L
Linus Torvalds 已提交
1583 1584
	}

1585
	oif = arg->bound_dev_if;
1586 1587
	if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
		oif = skb->skb_iif;
1588 1589

	flowi4_init_output(&fl4, oif,
1590
			   IP4_REPLY_MARK(net, skb->mark),
1591
			   RT_TOS(arg->tos),
1592
			   RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol,
1593
			   ip_reply_arg_flowi_flags(arg),
1594
			   daddr, saddr,
1595 1596
			   tcp_hdr(skb)->source, tcp_hdr(skb)->dest,
			   arg->uid);
1597
	security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
1598
	rt = ip_route_output_key(net, &fl4);
1599 1600
	if (IS_ERR(rt))
		return;
L
Linus Torvalds 已提交
1601

1602
	inet_sk(sk)->tos = arg->tos;
L
Linus Torvalds 已提交
1603 1604

	sk->sk_priority = skb->priority;
1605
	sk->sk_protocol = ip_hdr(skb)->protocol;
1606
	sk->sk_bound_dev_if = arg->bound_dev_if;
1607
	sk->sk_sndbuf = sysctl_wmem_default;
1608 1609 1610 1611 1612 1613 1614
	err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base,
			     len, 0, &ipc, &rt, MSG_DONTWAIT);
	if (unlikely(err)) {
		ip_flush_pending_frames(sk);
		goto out;
	}

1615 1616
	nskb = skb_peek(&sk->sk_write_queue);
	if (nskb) {
L
Linus Torvalds 已提交
1617
		if (arg->csumoffset >= 0)
1618 1619
			*((__sum16 *)skb_transport_header(nskb) +
			  arg->csumoffset) = csum_fold(csum_add(nskb->csum,
1620
								arg->csum));
1621
		nskb->ip_summed = CHECKSUM_NONE;
1622
		ip_push_pending_frames(sk, &fl4);
L
Linus Torvalds 已提交
1623
	}
1624
out:
L
Linus Torvalds 已提交
1625 1626 1627 1628 1629 1630 1631 1632
	ip_rt_put(rt);
}

void __init ip_init(void)
{
	ip_rt_init();
	inet_initpeers();

1633 1634
#if defined(CONFIG_IP_MULTICAST)
	igmp_mc_init();
L
Linus Torvalds 已提交
1635 1636
#endif
}