ip_output.c 39.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		The Internet Protocol (IP) output module.
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Donald Becker, <becker@super.org>
 *		Alan Cox, <Alan.Cox@linux.org>
 *		Richard Underwood
 *		Stefan Becker, <stefanb@yello.ping.de>
 *		Jorge Cwik, <jorge@laser.satlink.net>
 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *		Hirokazu Takahashi, <taka@valinux.co.jp>
 *
 *	See ip_input.c for original log
 *
 *	Fixes:
 *		Alan Cox	:	Missing nonblock feature in ip_build_xmit.
 *		Mike Kilburn	:	htons() missing in ip_build_xmit.
23
 *		Bradford Johnson:	Fix faulty handling of some frames when
L
Linus Torvalds 已提交
24 25 26 27 28 29 30 31 32 33
 *					no route is found.
 *		Alexander Demenshin:	Missing sk/skb free in ip_queue_xmit
 *					(in case if packet not accepted by
 *					output firewall rules)
 *		Mike McLagan	:	Routing by source
 *		Alexey Kuznetsov:	use new route cache
 *		Andi Kleen:		Fix broken PMTU recovery and remove
 *					some redundant tests.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
 *		Andi Kleen	: 	Replace ip_reply with ip_send_reply.
34 35 36
 *		Andi Kleen	:	Split fast and slow ip_build_xmit path
 *					for decreased register pressure on x86
 *					and more readibility.
L
Linus Torvalds 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
 *		Marc Boucher	:	When call_out_firewall returns FW_QUEUE,
 *					silently drop skb instead of failing with -EPERM.
 *		Detlev Wengorz	:	Copy protocol for fragments.
 *		Hirokazu Takahashi:	HW checksumming for outgoing UDP
 *					datagrams.
 *		Hirokazu Takahashi:	sendfile() on UDP works now.
 */

#include <asm/uaccess.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/errno.h>
A
Al Viro 已提交
52
#include <linux/highmem.h>
53
#include <linux/slab.h>
L
Linus Torvalds 已提交
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68

#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/init.h>

#include <net/snmp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
69
#include <net/xfrm.h>
L
Linus Torvalds 已提交
70 71 72 73 74 75
#include <linux/skbuff.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/icmp.h>
#include <net/checksum.h>
#include <net/inetpeer.h>
76
#include <net/lwtunnel.h>
L
Linus Torvalds 已提交
77 78 79 80
#include <linux/igmp.h>
#include <linux/netfilter_ipv4.h>
#include <linux/netfilter_bridge.h>
#include <linux/netlink.h>
81
#include <linux/tcp.h>
L
Linus Torvalds 已提交
82

83 84 85 86
static int
ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
	    unsigned int mtu,
	    int (*output)(struct net *, struct sock *, struct sk_buff *));
87

L
Linus Torvalds 已提交
88
/* Generate a checksum for an outgoing IP datagram. */
89
void ip_send_check(struct iphdr *iph)
L
Linus Torvalds 已提交
90 91 92 93
{
	iph->check = 0;
	iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
}
E
Eric Dumazet 已提交
94
EXPORT_SYMBOL(ip_send_check);
L
Linus Torvalds 已提交
95

96
int __ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
H
Herbert Xu 已提交
97 98 99 100 101
{
	struct iphdr *iph = ip_hdr(skb);

	iph->tot_len = htons(skb->len);
	ip_send_check(iph);
102 103 104 105 106 107 108 109

	/* if egress device is enslaved to an L3 master device pass the
	 * skb to its handler for processing
	 */
	skb = l3mdev_ip_out(sk, skb);
	if (unlikely(!skb))
		return 0;

110 111
	return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT,
		       net, sk, skb, NULL, skb_dst(skb)->dev,
112
		       dst_output);
113 114
}

115
int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
H
Herbert Xu 已提交
116 117 118
{
	int err;

119
	err = __ip_local_out(net, sk, skb);
H
Herbert Xu 已提交
120
	if (likely(err == 1))
121
		err = dst_output(net, sk, skb);
H
Herbert Xu 已提交
122 123 124

	return err;
}
125
EXPORT_SYMBOL_GPL(ip_local_out);
H
Herbert Xu 已提交
126

L
Linus Torvalds 已提交
127 128 129 130 131
static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
{
	int ttl = inet->uc_ttl;

	if (ttl < 0)
132
		ttl = ip4_dst_hoplimit(dst);
L
Linus Torvalds 已提交
133 134 135
	return ttl;
}

136
/*
L
Linus Torvalds 已提交
137 138 139
 *		Add an ip header to a skbuff and send it out.
 *
 */
140
int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk,
141
			  __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)
L
Linus Torvalds 已提交
142 143
{
	struct inet_sock *inet = inet_sk(sk);
E
Eric Dumazet 已提交
144
	struct rtable *rt = skb_rtable(skb);
145
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
146 147 148
	struct iphdr *iph;

	/* Build the IP header. */
149
	skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
150
	skb_reset_network_header(skb);
151
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
152 153 154
	iph->version  = 4;
	iph->ihl      = 5;
	iph->tos      = inet->tos;
155
	iph->ttl      = ip_select_ttl(inet, &rt->dst);
156 157
	iph->daddr    = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
	iph->saddr    = saddr;
L
Linus Torvalds 已提交
158
	iph->protocol = sk->sk_protocol;
159 160 161 162 163
	if (ip_dont_fragment(sk, &rt->dst)) {
		iph->frag_off = htons(IP_DF);
		iph->id = 0;
	} else {
		iph->frag_off = 0;
164
		__ip_select_ident(net, iph, 1);
165
	}
L
Linus Torvalds 已提交
166

167 168 169
	if (opt && opt->opt.optlen) {
		iph->ihl += opt->opt.optlen>>2;
		ip_options_build(skb, &opt->opt, daddr, rt, 0);
L
Linus Torvalds 已提交
170 171 172
	}

	skb->priority = sk->sk_priority;
173
	skb->mark = sk->sk_mark;
L
Linus Torvalds 已提交
174 175

	/* Send it out. */
176
	return ip_local_out(net, skb->sk, skb);
L
Linus Torvalds 已提交
177
}
178 179
EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);

180
static int ip_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
181
{
E
Eric Dumazet 已提交
182
	struct dst_entry *dst = skb_dst(skb);
183
	struct rtable *rt = (struct rtable *)dst;
L
Linus Torvalds 已提交
184
	struct net_device *dev = dst->dev;
185
	unsigned int hh_len = LL_RESERVED_SPACE(dev);
186
	struct neighbour *neigh;
187
	u32 nexthop;
L
Linus Torvalds 已提交
188

189
	if (rt->rt_type == RTN_MULTICAST) {
190
		IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTMCAST, skb->len);
191
	} else if (rt->rt_type == RTN_BROADCAST)
192
		IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTBCAST, skb->len);
193

L
Linus Torvalds 已提交
194
	/* Be paranoid, rather than too clever. */
195
	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
L
Linus Torvalds 已提交
196 197 198
		struct sk_buff *skb2;

		skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
199
		if (!skb2) {
L
Linus Torvalds 已提交
200 201 202 203 204
			kfree_skb(skb);
			return -ENOMEM;
		}
		if (skb->sk)
			skb_set_owner_w(skb2, skb->sk);
205
		consume_skb(skb);
L
Linus Torvalds 已提交
206 207 208
		skb = skb2;
	}

209 210 211 212 213 214 215
	if (lwtunnel_xmit_redirect(dst->lwtstate)) {
		int res = lwtunnel_xmit(skb);

		if (res < 0 || res == LWTUNNEL_XMIT_DONE)
			return res;
	}

216
	rcu_read_lock_bh();
J
Julian Anastasov 已提交
217
	nexthop = (__force u32) rt_nexthop(rt, ip_hdr(skb)->daddr);
218 219 220
	neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
	if (unlikely(!neigh))
		neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
221
	if (!IS_ERR(neigh)) {
222
		int res = dst_neigh_output(dst, neigh, skb);
223

224
		rcu_read_unlock_bh();
225 226
		return res;
	}
227
	rcu_read_unlock_bh();
228

229 230
	net_dbg_ratelimited("%s: No header cache and no neighbour!\n",
			    __func__);
L
Linus Torvalds 已提交
231 232 233 234
	kfree_skb(skb);
	return -EINVAL;
}

235 236
static int ip_finish_output_gso(struct net *net, struct sock *sk,
				struct sk_buff *skb, unsigned int mtu)
237 238 239 240 241
{
	netdev_features_t features;
	struct sk_buff *segs;
	int ret = 0;

242
	/* common case: seglen is <= mtu
243
	 */
244
	if (skb_gso_validate_mtu(skb, mtu))
245
		return ip_finish_output2(net, sk, skb);
246 247 248 249 250 251 252 253 254

	/* Slowpath -  GSO segment length is exceeding the dst MTU.
	 *
	 * This can happen in two cases:
	 * 1) TCP GRO packet, DF bit not set
	 * 2) skb arrived via virtio-net, we thus get TSO/GSO skbs directly
	 * from host network stack.
	 */
	features = netif_skb_features(skb);
255
	BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_SGO_CB_OFFSET);
256
	segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
257
	if (IS_ERR_OR_NULL(segs)) {
258 259 260 261 262 263 264 265 266 267 268
		kfree_skb(skb);
		return -ENOMEM;
	}

	consume_skb(skb);

	do {
		struct sk_buff *nskb = segs->next;
		int err;

		segs->next = NULL;
269
		err = ip_fragment(net, sk, segs, mtu, ip_finish_output2);
270 271 272 273 274 275 276 277 278

		if (err && ret == 0)
			ret = err;
		segs = nskb;
	} while (segs);

	return ret;
}

279
static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
280
{
281 282
	unsigned int mtu;

283 284
#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
	/* Policy lookup after SNAT yielded a new policy */
285
	if (skb_dst(skb)->xfrm) {
286
		IPCB(skb)->flags |= IPSKB_REROUTED;
287
		return dst_output(net, sk, skb);
288
	}
289
#endif
290
	mtu = ip_skb_dst_mtu(sk, skb);
291
	if (skb_is_gso(skb))
292
		return ip_finish_output_gso(net, sk, skb, mtu);
293

294
	if (skb->len > mtu || (IPCB(skb)->flags & IPSKB_FRAG_PMTU))
295
		return ip_fragment(net, sk, skb, mtu, ip_finish_output2);
296

297
	return ip_finish_output2(net, sk, skb);
L
Linus Torvalds 已提交
298 299
}

E
Eric W. Biederman 已提交
300
int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
301
{
E
Eric Dumazet 已提交
302
	struct rtable *rt = skb_rtable(skb);
303
	struct net_device *dev = rt->dst.dev;
L
Linus Torvalds 已提交
304 305 306 307

	/*
	 *	If the indicated interface is up and running, send the packet.
	 */
308
	IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
L
Linus Torvalds 已提交
309 310 311 312 313 314 315 316 317

	skb->dev = dev;
	skb->protocol = htons(ETH_P_IP);

	/*
	 *	Multicasts are looped back for other local users
	 */

	if (rt->rt_flags&RTCF_MULTICAST) {
318
		if (sk_mc_loop(sk)
L
Linus Torvalds 已提交
319 320 321 322 323 324 325 326 327
#ifdef CONFIG_IP_MROUTE
		/* Small optimization: do not loopback not local frames,
		   which returned after forwarding; they will be  dropped
		   by ip_mr_input in any case.
		   Note, that local frames are looped back to be delivered
		   to local recipients.

		   This check is duplicated in ip_mr_input at the moment.
		 */
328 329 330
		    &&
		    ((rt->rt_flags & RTCF_LOCAL) ||
		     !(IPCB(skb)->flags & IPSKB_FORWARDED))
L
Linus Torvalds 已提交
331
#endif
332
		   ) {
L
Linus Torvalds 已提交
333 334
			struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
			if (newskb)
335
				NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
336
					net, sk, newskb, NULL, newskb->dev,
337
					dev_loopback_xmit);
L
Linus Torvalds 已提交
338 339 340 341
		}

		/* Multicasts with ttl 0 must not go beyond the host */

342
		if (ip_hdr(skb)->ttl == 0) {
L
Linus Torvalds 已提交
343 344 345 346 347 348 349 350
			kfree_skb(skb);
			return 0;
		}
	}

	if (rt->rt_flags&RTCF_BROADCAST) {
		struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
		if (newskb)
351 352 353
			NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
				net, sk, newskb, NULL, newskb->dev,
				dev_loopback_xmit);
L
Linus Torvalds 已提交
354 355
	}

356 357 358
	return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
			    net, sk, skb, NULL, skb->dev,
			    ip_finish_output,
359
			    !(IPCB(skb)->flags & IPSKB_REROUTED));
L
Linus Torvalds 已提交
360 361
}

E
Eric W. Biederman 已提交
362
int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
363
{
E
Eric Dumazet 已提交
364
	struct net_device *dev = skb_dst(skb)->dev;
365

366
	IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
L
Linus Torvalds 已提交
367

368 369 370
	skb->dev = dev;
	skb->protocol = htons(ETH_P_IP);

371 372
	return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
			    net, sk, skb, NULL, dev,
373
			    ip_finish_output,
374
			    !(IPCB(skb)->flags & IPSKB_REROUTED));
L
Linus Torvalds 已提交
375 376
}

377 378 379 380 381 382 383 384 385 386 387 388 389 390
/*
 * copy saddr and daddr, possibly using 64bit load/stores
 * Equivalent to :
 *   iph->saddr = fl4->saddr;
 *   iph->daddr = fl4->daddr;
 */
static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4)
{
	BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) !=
		     offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr));
	memcpy(&iph->saddr, &fl4->saddr,
	       sizeof(fl4->saddr) + sizeof(fl4->daddr));
}

391 392
/* Note: skb->sk can be different from sk, in case of tunnels */
int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
L
Linus Torvalds 已提交
393 394
{
	struct inet_sock *inet = inet_sk(sk);
395
	struct net *net = sock_net(sk);
396
	struct ip_options_rcu *inet_opt;
397
	struct flowi4 *fl4;
L
Linus Torvalds 已提交
398 399
	struct rtable *rt;
	struct iphdr *iph;
400
	int res;
L
Linus Torvalds 已提交
401 402 403 404

	/* Skip all of this if the packet is already routed,
	 * f.e. by something like SCTP.
	 */
405
	rcu_read_lock();
406
	inet_opt = rcu_dereference(inet->inet_opt);
407
	fl4 = &fl->u.ip4;
E
Eric Dumazet 已提交
408
	rt = skb_rtable(skb);
409
	if (rt)
L
Linus Torvalds 已提交
410 411 412 413
		goto packet_routed;

	/* Make sure we can route this packet. */
	rt = (struct rtable *)__sk_dst_check(sk, 0);
414
	if (!rt) {
A
Al Viro 已提交
415
		__be32 daddr;
L
Linus Torvalds 已提交
416 417

		/* Use correct destination address if we have options. */
E
Eric Dumazet 已提交
418
		daddr = inet->inet_daddr;
419 420
		if (inet_opt && inet_opt->opt.srr)
			daddr = inet_opt->opt.faddr;
L
Linus Torvalds 已提交
421

422 423 424 425
		/* If this fails, retransmit mechanism of transport layer will
		 * keep trying until route appears or the connection times
		 * itself out.
		 */
426
		rt = ip_route_output_ports(net, fl4, sk,
427 428 429 430 431 432 433 434
					   daddr, inet->inet_saddr,
					   inet->inet_dport,
					   inet->inet_sport,
					   sk->sk_protocol,
					   RT_CONN_FLAGS(sk),
					   sk->sk_bound_dev_if);
		if (IS_ERR(rt))
			goto no_route;
435
		sk_setup_caps(sk, &rt->dst);
L
Linus Torvalds 已提交
436
	}
437
	skb_dst_set_noref(skb, &rt->dst);
L
Linus Torvalds 已提交
438 439

packet_routed:
J
Julian Anastasov 已提交
440
	if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway)
L
Linus Torvalds 已提交
441 442 443
		goto no_route;

	/* OK, we know where to send it, allocate and build IP header. */
444
	skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
445
	skb_reset_network_header(skb);
446
	iph = ip_hdr(skb);
447
	*((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
W
WANG Cong 已提交
448
	if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df)
L
Linus Torvalds 已提交
449 450 451
		iph->frag_off = htons(IP_DF);
	else
		iph->frag_off = 0;
452
	iph->ttl      = ip_select_ttl(inet, &rt->dst);
L
Linus Torvalds 已提交
453
	iph->protocol = sk->sk_protocol;
454 455
	ip_copy_addrs(iph, fl4);

L
Linus Torvalds 已提交
456 457
	/* Transport layer set skb->h.foo itself. */

458 459 460
	if (inet_opt && inet_opt->opt.optlen) {
		iph->ihl += inet_opt->opt.optlen >> 2;
		ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
L
Linus Torvalds 已提交
461 462
	}

463
	ip_select_ident_segs(net, skb, sk,
464
			     skb_shinfo(skb)->gso_segs ?: 1);
L
Linus Torvalds 已提交
465

466
	/* TODO : should we use skb->sk here instead of sk ? */
L
Linus Torvalds 已提交
467
	skb->priority = sk->sk_priority;
468
	skb->mark = sk->sk_mark;
L
Linus Torvalds 已提交
469

470
	res = ip_local_out(net, sk, skb);
471 472
	rcu_read_unlock();
	return res;
L
Linus Torvalds 已提交
473 474

no_route:
475
	rcu_read_unlock();
476
	IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
L
Linus Torvalds 已提交
477 478 479
	kfree_skb(skb);
	return -EHOSTUNREACH;
}
E
Eric Dumazet 已提交
480
EXPORT_SYMBOL(ip_queue_xmit);
L
Linus Torvalds 已提交
481 482 483 484 485 486

static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
{
	to->pkt_type = from->pkt_type;
	to->priority = from->priority;
	to->protocol = from->protocol;
E
Eric Dumazet 已提交
487
	skb_dst_drop(to);
488
	skb_dst_copy(to, from);
L
Linus Torvalds 已提交
489
	to->dev = from->dev;
T
Thomas Graf 已提交
490
	to->mark = from->mark;
L
Linus Torvalds 已提交
491 492 493 494 495 496 497

	/* Copy the flags to each fragment. */
	IPCB(to)->flags = IPCB(from)->flags;

#ifdef CONFIG_NET_SCHED
	to->tc_index = from->tc_index;
#endif
498
	nf_copy(to, from);
499
#if IS_ENABLED(CONFIG_IP_VS)
500
	to->ipvs_property = from->ipvs_property;
L
Linus Torvalds 已提交
501
#endif
502
	skb_copy_secmark(to, from);
L
Linus Torvalds 已提交
503 504
}

505
static int ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
506
		       unsigned int mtu,
507
		       int (*output)(struct net *, struct sock *, struct sk_buff *))
508 509 510
{
	struct iphdr *iph = ip_hdr(skb);

511
	if ((iph->frag_off & htons(IP_DF)) == 0)
512
		return ip_do_fragment(net, sk, skb, output);
513 514

	if (unlikely(!skb->ignore_df ||
515 516
		     (IPCB(skb)->frag_max_size &&
		      IPCB(skb)->frag_max_size > mtu))) {
517
		IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
518 519 520 521 522 523
		icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
			  htonl(mtu));
		kfree_skb(skb);
		return -EMSGSIZE;
	}

524
	return ip_do_fragment(net, sk, skb, output);
525 526
}

L
Linus Torvalds 已提交
527 528 529 530 531 532 533
/*
 *	This IP datagram is too large to be sent in one piece.  Break it up into
 *	smaller pieces (each of size equal to IP header plus
 *	a block of the data of the original IP data part) that will yet fit in a
 *	single device frame, and queue such a frame for sending.
 */

534 535
int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
		   int (*output)(struct net *, struct sock *, struct sk_buff *))
L
Linus Torvalds 已提交
536 537 538 539
{
	struct iphdr *iph;
	int ptr;
	struct sk_buff *skb2;
540
	unsigned int mtu, hlen, left, len, ll_rs;
L
Linus Torvalds 已提交
541
	int offset;
542
	__be16 not_last_frag;
E
Eric Dumazet 已提交
543
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
544 545
	int err = 0;

546 547 548 549 550
	/* for offloaded checksums cleanup checksum before fragmentation */
	if (skb->ip_summed == CHECKSUM_PARTIAL &&
	    (err = skb_checksum_help(skb)))
		goto fail;

L
Linus Torvalds 已提交
551 552 553 554
	/*
	 *	Point into the IP datagram header.
	 */

555
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
556

557
	mtu = ip_skb_dst_mtu(sk, skb);
558 559
	if (IPCB(skb)->frag_max_size && IPCB(skb)->frag_max_size < mtu)
		mtu = IPCB(skb)->frag_max_size;
L
Linus Torvalds 已提交
560 561 562 563 564 565

	/*
	 *	Setup starting values.
	 */

	hlen = iph->ihl * 4;
566
	mtu = mtu - hlen;	/* Size of data space */
H
Herbert Xu 已提交
567
	IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
L
Linus Torvalds 已提交
568 569 570 571 572 573 574 575

	/* When frag_list is given, use it. First, check its validity:
	 * some transformers could create wrong frag_list or break existing
	 * one, it is not prohibited. In this case fall back to copying.
	 *
	 * LATER: this step can be merged to real generation of fragments,
	 * we can switch to copy when see the first bad fragment.
	 */
576
	if (skb_has_frag_list(skb)) {
577
		struct sk_buff *frag, *frag2;
L
Linus Torvalds 已提交
578 579 580 581
		int first_len = skb_pagelen(skb);

		if (first_len - hlen > mtu ||
		    ((first_len - hlen) & 7) ||
582
		    ip_is_fragment(iph) ||
L
Linus Torvalds 已提交
583 584 585
		    skb_cloned(skb))
			goto slow_path;

586
		skb_walk_frags(skb, frag) {
L
Linus Torvalds 已提交
587 588 589 590
			/* Correct geometry. */
			if (frag->len > mtu ||
			    ((frag->len & 7) && frag->next) ||
			    skb_headroom(frag) < hlen)
591
				goto slow_path_clean;
L
Linus Torvalds 已提交
592 593 594

			/* Partially cloned skb? */
			if (skb_shared(frag))
595
				goto slow_path_clean;
596 597 598 599 600 601

			BUG_ON(frag->sk);
			if (skb->sk) {
				frag->sk = skb->sk;
				frag->destructor = sock_wfree;
			}
602
			skb->truesize -= frag->truesize;
L
Linus Torvalds 已提交
603 604 605 606 607 608 609
		}

		/* Everything is OK. Generate! */

		err = 0;
		offset = 0;
		frag = skb_shinfo(skb)->frag_list;
610
		skb_frag_list_init(skb);
L
Linus Torvalds 已提交
611 612 613 614 615 616 617 618 619 620 621
		skb->data_len = first_len - skb_headlen(skb);
		skb->len = first_len;
		iph->tot_len = htons(first_len);
		iph->frag_off = htons(IP_MF);
		ip_send_check(iph);

		for (;;) {
			/* Prepare header of the next frame,
			 * before previous one went down. */
			if (frag) {
				frag->ip_summed = CHECKSUM_NONE;
622
				skb_reset_transport_header(frag);
623 624
				__skb_push(frag, hlen);
				skb_reset_network_header(frag);
625
				memcpy(skb_network_header(frag), iph, hlen);
626
				iph = ip_hdr(frag);
L
Linus Torvalds 已提交
627 628 629 630 631 632
				iph->tot_len = htons(frag->len);
				ip_copy_metadata(frag, skb);
				if (offset == 0)
					ip_options_fragment(frag);
				offset += skb->len - hlen;
				iph->frag_off = htons(offset>>3);
633
				if (frag->next)
L
Linus Torvalds 已提交
634 635 636 637 638
					iph->frag_off |= htons(IP_MF);
				/* Ready, complete checksum */
				ip_send_check(iph);
			}

639
			err = output(net, sk, skb);
L
Linus Torvalds 已提交
640

641
			if (!err)
642
				IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
643 644 645 646 647 648 649 650 651
			if (err || !frag)
				break;

			skb = frag;
			frag = skb->next;
			skb->next = NULL;
		}

		if (err == 0) {
652
			IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
L
Linus Torvalds 已提交
653 654 655 656 657 658 659 660
			return 0;
		}

		while (frag) {
			skb = frag->next;
			kfree_skb(frag);
			frag = skb;
		}
661
		IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
662
		return err;
663 664 665 666 667 668 669 670 671

slow_path_clean:
		skb_walk_frags(skb, frag2) {
			if (frag2 == frag)
				break;
			frag2->sk = NULL;
			frag2->destructor = NULL;
			skb->truesize += frag2->truesize;
		}
L
Linus Torvalds 已提交
672 673 674
	}

slow_path:
675
	iph = ip_hdr(skb);
676

L
Linus Torvalds 已提交
677
	left = skb->len - hlen;		/* Space per frame */
678
	ptr = hlen;		/* Where to start from */
L
Linus Torvalds 已提交
679

680
	ll_rs = LL_RESERVED_SPACE(rt->dst.dev);
681

L
Linus Torvalds 已提交
682 683 684 685 686 687 688 689 690 691 692
	/*
	 *	Fragment the datagram.
	 */

	offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
	not_last_frag = iph->frag_off & htons(IP_MF);

	/*
	 *	Keep copying data until we run out.
	 */

S
Stephen Hemminger 已提交
693
	while (left > 0) {
L
Linus Torvalds 已提交
694 695 696 697
		len = left;
		/* IF: it doesn't fit, use 'mtu' - the data space left */
		if (len > mtu)
			len = mtu;
L
Lucas De Marchi 已提交
698
		/* IF: we are not sending up to and including the packet end
L
Linus Torvalds 已提交
699 700 701 702 703
		   then align the next start on an eight byte boundary */
		if (len < left)	{
			len &= ~7;
		}

704 705 706
		/* Allocate buffer */
		skb2 = alloc_skb(len + hlen + ll_rs, GFP_ATOMIC);
		if (!skb2) {
L
Linus Torvalds 已提交
707 708 709 710 711 712 713 714 715 716 717
			err = -ENOMEM;
			goto fail;
		}

		/*
		 *	Set up data on packet
		 */

		ip_copy_metadata(skb2, skb);
		skb_reserve(skb2, ll_rs);
		skb_put(skb2, len + hlen);
718
		skb_reset_network_header(skb2);
719
		skb2->transport_header = skb2->network_header + hlen;
L
Linus Torvalds 已提交
720 721 722 723 724 725 726 727 728 729 730 731 732

		/*
		 *	Charge the memory for the fragment to any owner
		 *	it might possess
		 */

		if (skb->sk)
			skb_set_owner_w(skb2, skb->sk);

		/*
		 *	Copy the packet header into the new buffer.
		 */

733
		skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
L
Linus Torvalds 已提交
734 735 736 737

		/*
		 *	Copy a block of the IP datagram.
		 */
738
		if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
L
Linus Torvalds 已提交
739 740 741 742 743 744
			BUG();
		left -= len;

		/*
		 *	Fill in the new header fields.
		 */
745
		iph = ip_hdr(skb2);
L
Linus Torvalds 已提交
746 747
		iph->frag_off = htons((offset >> 3));

748 749 750
		if (IPCB(skb)->flags & IPSKB_FRAG_PMTU)
			iph->frag_off |= htons(IP_DF);

L
Linus Torvalds 已提交
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
		/* ANK: dirty, but effective trick. Upgrade options only if
		 * the segment to be fragmented was THE FIRST (otherwise,
		 * options are already fixed) and make it ONCE
		 * on the initial skb, so that all the following fragments
		 * will inherit fixed options.
		 */
		if (offset == 0)
			ip_options_fragment(skb);

		/*
		 *	Added AC : If we are fragmenting a fragment that's not the
		 *		   last fragment then keep MF on each bit
		 */
		if (left > 0 || not_last_frag)
			iph->frag_off |= htons(IP_MF);
		ptr += len;
		offset += len;

		/*
		 *	Put this fragment into the sending queue.
		 */
		iph->tot_len = htons(len + hlen);

		ip_send_check(iph);

776
		err = output(net, sk, skb2);
L
Linus Torvalds 已提交
777 778
		if (err)
			goto fail;
779

780
		IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
781
	}
782
	consume_skb(skb);
783
	IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
L
Linus Torvalds 已提交
784 785 786
	return err;

fail:
787
	kfree_skb(skb);
788
	IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
789 790
	return err;
}
791
EXPORT_SYMBOL(ip_do_fragment);
792

L
Linus Torvalds 已提交
793 794 795
int
ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
{
796
	struct msghdr *msg = from;
L
Linus Torvalds 已提交
797

798
	if (skb->ip_summed == CHECKSUM_PARTIAL) {
799
		if (copy_from_iter(to, len, &msg->msg_iter) != len)
L
Linus Torvalds 已提交
800 801
			return -EFAULT;
	} else {
802
		__wsum csum = 0;
803
		if (csum_and_copy_from_iter(to, len, &csum, &msg->msg_iter) != len)
L
Linus Torvalds 已提交
804 805 806 807 808
			return -EFAULT;
		skb->csum = csum_block_add(skb->csum, csum, odd);
	}
	return 0;
}
E
Eric Dumazet 已提交
809
EXPORT_SYMBOL(ip_generic_getfrag);
L
Linus Torvalds 已提交
810

811
static inline __wsum
L
Linus Torvalds 已提交
812 813 814
csum_page(struct page *page, int offset, int copy)
{
	char *kaddr;
815
	__wsum csum;
L
Linus Torvalds 已提交
816 817 818 819 820 821
	kaddr = kmap(page);
	csum = csum_partial(kaddr + offset, copy, 0);
	kunmap(page);
	return csum;
}

A
Adrian Bunk 已提交
822
static inline int ip_ufo_append_data(struct sock *sk,
823
			struct sk_buff_head *queue,
824 825 826
			int getfrag(void *from, char *to, int offset, int len,
			       int odd, struct sk_buff *skb),
			void *from, int length, int hh_len, int fragheaderlen,
827
			int transhdrlen, int maxfraglen, unsigned int flags)
828 829 830 831 832 833 834 835
{
	struct sk_buff *skb;
	int err;

	/* There is support for UDP fragmentation offload by network
	 * device, so create one single skb packet containing complete
	 * udp datagram
	 */
836 837
	skb = skb_peek_tail(queue);
	if (!skb) {
838 839 840 841
		skb = sock_alloc_send_skb(sk,
			hh_len + fragheaderlen + transhdrlen + 20,
			(flags & MSG_DONTWAIT), &err);

842
		if (!skb)
843 844 845 846 847 848
			return err;

		/* reserve space for Hardware header */
		skb_reserve(skb, hh_len);

		/* create space for UDP/IP header */
849
		skb_put(skb, fragheaderlen + transhdrlen);
850 851

		/* initialize network header pointer */
852
		skb_reset_network_header(skb);
853 854

		/* initialize protocol header pointer */
855
		skb->transport_header = skb->network_header + fragheaderlen;
856 857 858

		skb->csum = 0;

859
		__skb_queue_tail(queue, skb);
860 861
	} else if (skb_is_gso(skb)) {
		goto append;
862
	}
863

864 865 866 867 868 869
	skb->ip_summed = CHECKSUM_PARTIAL;
	/* specify the length of each IP datagram fragment */
	skb_shinfo(skb)->gso_size = maxfraglen - fragheaderlen;
	skb_shinfo(skb)->gso_type = SKB_GSO_UDP;

append:
870 871
	return skb_append_datato_frags(sk, skb, getfrag, from,
				       (length - transhdrlen));
872 873
}

874 875 876
static int __ip_append_data(struct sock *sk,
			    struct flowi4 *fl4,
			    struct sk_buff_head *queue,
877
			    struct inet_cork *cork,
878
			    struct page_frag *pfrag,
879 880 881 882
			    int getfrag(void *from, char *to, int offset,
					int len, int odd, struct sk_buff *skb),
			    void *from, int length, int transhdrlen,
			    unsigned int flags)
L
Linus Torvalds 已提交
883 884 885 886
{
	struct inet_sock *inet = inet_sk(sk);
	struct sk_buff *skb;

887
	struct ip_options *opt = cork->opt;
L
Linus Torvalds 已提交
888 889 890 891 892 893
	int hh_len;
	int exthdrlen;
	int mtu;
	int copy;
	int err;
	int offset = 0;
894
	unsigned int maxfraglen, fragheaderlen, maxnonfragsize;
L
Linus Torvalds 已提交
895
	int csummode = CHECKSUM_NONE;
896
	struct rtable *rt = (struct rtable *)cork->dst;
897
	u32 tskey = 0;
L
Linus Torvalds 已提交
898

899 900 901
	skb = skb_peek_tail(queue);

	exthdrlen = !skb ? rt->dst.header_len : 0;
902
	mtu = cork->fragsize;
903 904 905
	if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&
	    sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
		tskey = sk->sk_tskey++;
L
Linus Torvalds 已提交
906

907
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
L
Linus Torvalds 已提交
908 909 910

	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
W
WANG Cong 已提交
911
	maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu;
L
Linus Torvalds 已提交
912

913
	if (cork->length + length > maxnonfragsize - fragheaderlen) {
914
		ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
915
			       mtu - (opt ? opt->optlen : 0));
L
Linus Torvalds 已提交
916 917 918 919 920 921 922 923 924
		return -EMSGSIZE;
	}

	/*
	 * transhdrlen > 0 means that this is the first fragment and we wish
	 * it won't be fragmented in the future.
	 */
	if (transhdrlen &&
	    length + fragheaderlen <= mtu &&
925
	    rt->dst.dev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM) &&
926
	    !(flags & MSG_MORE) &&
L
Linus Torvalds 已提交
927
	    !exthdrlen)
928
		csummode = CHECKSUM_PARTIAL;
L
Linus Torvalds 已提交
929

930
	cork->length += length;
931
	if (((length > mtu) || (skb && skb_is_gso(skb))) &&
932
	    (sk->sk_protocol == IPPROTO_UDP) &&
933
	    (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len &&
934
	    (sk->sk_type == SOCK_DGRAM) && !sk->sk_no_check_tx) {
935 936
		err = ip_ufo_append_data(sk, queue, getfrag, from, length,
					 hh_len, fragheaderlen, transhdrlen,
937
					 maxfraglen, flags);
938
		if (err)
939 940 941
			goto error;
		return 0;
	}
L
Linus Torvalds 已提交
942 943 944 945 946 947 948 949

	/* So, what's going on in the loop below?
	 *
	 * We use calculated fragment length to generate chained skb,
	 * each of segments is IP fragment ready for sending to network after
	 * adding appropriate IP header.
	 */

950
	if (!skb)
L
Linus Torvalds 已提交
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980
		goto alloc_new_skb;

	while (length > 0) {
		/* Check if the remaining data fits into current packet. */
		copy = mtu - skb->len;
		if (copy < length)
			copy = maxfraglen - skb->len;
		if (copy <= 0) {
			char *data;
			unsigned int datalen;
			unsigned int fraglen;
			unsigned int fraggap;
			unsigned int alloclen;
			struct sk_buff *skb_prev;
alloc_new_skb:
			skb_prev = skb;
			if (skb_prev)
				fraggap = skb_prev->len - maxfraglen;
			else
				fraggap = 0;

			/*
			 * If remaining data exceeds the mtu,
			 * we know we need more fragment(s).
			 */
			datalen = length + fraggap;
			if (datalen > mtu - fragheaderlen)
				datalen = maxfraglen - fragheaderlen;
			fraglen = datalen + fragheaderlen;

981
			if ((flags & MSG_MORE) &&
982
			    !(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
983 984
				alloclen = mtu;
			else
985
				alloclen = fraglen;
L
Linus Torvalds 已提交
986

987 988
			alloclen += exthdrlen;

L
Linus Torvalds 已提交
989 990 991 992 993
			/* The last fragment gets additional space at tail.
			 * Note, with MSG_MORE we overallocate on fragments,
			 * because we have no idea what fragment will be
			 * the last.
			 */
994
			if (datalen == length + fraggap)
995
				alloclen += rt->dst.trailer_len;
996

L
Linus Torvalds 已提交
997
			if (transhdrlen) {
998
				skb = sock_alloc_send_skb(sk,
L
Linus Torvalds 已提交
999 1000 1001 1002 1003 1004
						alloclen + hh_len + 15,
						(flags & MSG_DONTWAIT), &err);
			} else {
				skb = NULL;
				if (atomic_read(&sk->sk_wmem_alloc) <=
				    2 * sk->sk_sndbuf)
1005
					skb = sock_wmalloc(sk,
L
Linus Torvalds 已提交
1006 1007
							   alloclen + hh_len + 15, 1,
							   sk->sk_allocation);
1008
				if (unlikely(!skb))
L
Linus Torvalds 已提交
1009 1010
					err = -ENOBUFS;
			}
1011
			if (!skb)
L
Linus Torvalds 已提交
1012 1013 1014 1015 1016 1017 1018 1019
				goto error;

			/*
			 *	Fill in the control structures
			 */
			skb->ip_summed = csummode;
			skb->csum = 0;
			skb_reserve(skb, hh_len);
1020 1021

			/* only the initial fragment is time stamped */
1022
			skb_shinfo(skb)->tx_flags = cork->tx_flags;
1023
			cork->tx_flags = 0;
1024 1025
			skb_shinfo(skb)->tskey = tskey;
			tskey = 0;
L
Linus Torvalds 已提交
1026 1027 1028 1029

			/*
			 *	Find where to start putting bytes.
			 */
1030
			data = skb_put(skb, fraglen + exthdrlen);
1031
			skb_set_network_header(skb, exthdrlen);
1032 1033
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
1034
			data += fragheaderlen + exthdrlen;
L
Linus Torvalds 已提交
1035 1036 1037 1038 1039 1040 1041 1042

			if (fraggap) {
				skb->csum = skb_copy_and_csum_bits(
					skb_prev, maxfraglen,
					data + transhdrlen, fraggap, 0);
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
				data += fraggap;
1043
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
			}

			copy = datalen - transhdrlen - fraggap;
			if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
				err = -EFAULT;
				kfree_skb(skb);
				goto error;
			}

			offset += copy;
			length -= datalen - fraggap;
			transhdrlen = 0;
			exthdrlen = 0;
			csummode = CHECKSUM_NONE;

			/*
			 * Put the packet on the pending queue.
			 */
1062
			__skb_queue_tail(queue, skb);
L
Linus Torvalds 已提交
1063 1064 1065 1066 1067 1068
			continue;
		}

		if (copy > length)
			copy = length;

1069
		if (!(rt->dst.dev->features&NETIF_F_SG)) {
L
Linus Torvalds 已提交
1070 1071 1072
			unsigned int off;

			off = skb->len;
1073
			if (getfrag(from, skb_put(skb, copy),
L
Linus Torvalds 已提交
1074 1075 1076 1077 1078 1079 1080 1081
					offset, copy, off, skb) < 0) {
				__skb_trim(skb, off);
				err = -EFAULT;
				goto error;
			}
		} else {
			int i = skb_shinfo(skb)->nr_frags;

1082 1083
			err = -ENOMEM;
			if (!sk_page_frag_refill(sk, pfrag))
L
Linus Torvalds 已提交
1084
				goto error;
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095

			if (!skb_can_coalesce(skb, i, pfrag->page,
					      pfrag->offset)) {
				err = -EMSGSIZE;
				if (i == MAX_SKB_FRAGS)
					goto error;

				__skb_fill_page_desc(skb, i, pfrag->page,
						     pfrag->offset, 0);
				skb_shinfo(skb)->nr_frags = ++i;
				get_page(pfrag->page);
L
Linus Torvalds 已提交
1096
			}
1097 1098 1099 1100 1101 1102 1103 1104
			copy = min_t(int, copy, pfrag->size - pfrag->offset);
			if (getfrag(from,
				    page_address(pfrag->page) + pfrag->offset,
				    offset, copy, skb->len, skb) < 0)
				goto error_efault;

			pfrag->offset += copy;
			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
L
Linus Torvalds 已提交
1105 1106
			skb->len += copy;
			skb->data_len += copy;
1107 1108
			skb->truesize += copy;
			atomic_add(copy, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1109 1110 1111 1112 1113 1114 1115
		}
		offset += copy;
		length -= copy;
	}

	return 0;

1116 1117
error_efault:
	err = -EFAULT;
L
Linus Torvalds 已提交
1118
error:
1119
	cork->length -= length;
P
Pavel Emelyanov 已提交
1120
	IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
1121
	return err;
L
Linus Torvalds 已提交
1122 1123
}

1124 1125 1126
static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
			 struct ipcm_cookie *ipc, struct rtable **rtp)
{
1127
	struct ip_options_rcu *opt;
1128 1129 1130 1131 1132 1133 1134
	struct rtable *rt;

	/*
	 * setup for corking.
	 */
	opt = ipc->opt;
	if (opt) {
1135
		if (!cork->opt) {
1136 1137
			cork->opt = kmalloc(sizeof(struct ip_options) + 40,
					    sk->sk_allocation);
1138
			if (unlikely(!cork->opt))
1139 1140
				return -ENOBUFS;
		}
1141
		memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen);
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
		cork->flags |= IPCORK_OPT;
		cork->addr = ipc->addr;
	}
	rt = *rtp;
	if (unlikely(!rt))
		return -EFAULT;
	/*
	 * We steal reference to this route, caller should not release it
	 */
	*rtp = NULL;
1152 1153
	cork->fragsize = ip_sk_use_pmtu(sk) ?
			 dst_mtu(&rt->dst) : rt->dst.dev->mtu;
1154 1155
	cork->dst = &rt->dst;
	cork->length = 0;
1156 1157 1158
	cork->ttl = ipc->ttl;
	cork->tos = ipc->tos;
	cork->priority = ipc->priority;
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
	cork->tx_flags = ipc->tx_flags;

	return 0;
}

/*
 *	ip_append_data() and ip_append_page() can make one large IP datagram
 *	from many pieces of data. Each pieces will be holded on the socket
 *	until ip_push_pending_frames() is called. Each piece can be a page
 *	or non-page data.
 *
 *	Not only UDP, other transport protocols - e.g. raw sockets - can use
 *	this interface potentially.
 *
 *	LATER: length must be adjusted by pad at tail, when it is required.
 */
1175
int ip_append_data(struct sock *sk, struct flowi4 *fl4,
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
		   int getfrag(void *from, char *to, int offset, int len,
			       int odd, struct sk_buff *skb),
		   void *from, int length, int transhdrlen,
		   struct ipcm_cookie *ipc, struct rtable **rtp,
		   unsigned int flags)
{
	struct inet_sock *inet = inet_sk(sk);
	int err;

	if (flags&MSG_PROBE)
		return 0;

	if (skb_queue_empty(&sk->sk_write_queue)) {
1189
		err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp);
1190 1191 1192 1193 1194 1195
		if (err)
			return err;
	} else {
		transhdrlen = 0;
	}

1196 1197
	return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base,
				sk_page_frag(sk), getfrag,
1198 1199 1200
				from, length, transhdrlen, flags);
}

1201
ssize_t	ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,
L
Linus Torvalds 已提交
1202 1203 1204 1205 1206 1207
		       int offset, size_t size, int flags)
{
	struct inet_sock *inet = inet_sk(sk);
	struct sk_buff *skb;
	struct rtable *rt;
	struct ip_options *opt = NULL;
1208
	struct inet_cork *cork;
L
Linus Torvalds 已提交
1209 1210 1211 1212
	int hh_len;
	int mtu;
	int len;
	int err;
1213
	unsigned int maxfraglen, fragheaderlen, fraggap, maxnonfragsize;
L
Linus Torvalds 已提交
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223

	if (inet->hdrincl)
		return -EPERM;

	if (flags&MSG_PROBE)
		return 0;

	if (skb_queue_empty(&sk->sk_write_queue))
		return -EINVAL;

1224 1225 1226 1227
	cork = &inet->cork.base;
	rt = (struct rtable *)cork->dst;
	if (cork->flags & IPCORK_OPT)
		opt = cork->opt;
L
Linus Torvalds 已提交
1228

1229
	if (!(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
1230 1231
		return -EOPNOTSUPP;

1232
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1233
	mtu = cork->fragsize;
L
Linus Torvalds 已提交
1234 1235 1236

	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
W
WANG Cong 已提交
1237
	maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu;
L
Linus Torvalds 已提交
1238

1239
	if (cork->length + size > maxnonfragsize - fragheaderlen) {
1240 1241
		ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
			       mtu - (opt ? opt->optlen : 0));
L
Linus Torvalds 已提交
1242 1243 1244
		return -EMSGSIZE;
	}

1245 1246
	skb = skb_peek_tail(&sk->sk_write_queue);
	if (!skb)
L
Linus Torvalds 已提交
1247 1248
		return -EINVAL;

1249 1250
	if ((size + skb->len > mtu) &&
	    (sk->sk_protocol == IPPROTO_UDP) &&
1251
	    (rt->dst.dev->features & NETIF_F_UFO)) {
1252 1253 1254
		if (skb->ip_summed != CHECKSUM_PARTIAL)
			return -EOPNOTSUPP;

1255
		skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
H
Herbert Xu 已提交
1256
		skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1257
	}
1258
	cork->length += size;
1259

L
Linus Torvalds 已提交
1260
	while (size > 0) {
1261
		if (skb_is_gso(skb)) {
1262
			len = size;
1263
		} else {
1264 1265 1266 1267 1268 1269

			/* Check if the remaining data fits into current packet. */
			len = mtu - skb->len;
			if (len < size)
				len = maxfraglen - skb->len;
		}
L
Linus Torvalds 已提交
1270 1271 1272 1273 1274
		if (len <= 0) {
			struct sk_buff *skb_prev;
			int alloclen;

			skb_prev = skb;
1275
			fraggap = skb_prev->len - maxfraglen;
L
Linus Torvalds 已提交
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293

			alloclen = fragheaderlen + hh_len + fraggap + 15;
			skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
			if (unlikely(!skb)) {
				err = -ENOBUFS;
				goto error;
			}

			/*
			 *	Fill in the control structures
			 */
			skb->ip_summed = CHECKSUM_NONE;
			skb->csum = 0;
			skb_reserve(skb, hh_len);

			/*
			 *	Find where to start putting bytes.
			 */
1294
			skb_put(skb, fragheaderlen + fraggap);
1295
			skb_reset_network_header(skb);
1296 1297
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
L
Linus Torvalds 已提交
1298
			if (fraggap) {
1299 1300
				skb->csum = skb_copy_and_csum_bits(skb_prev,
								   maxfraglen,
1301
						    skb_transport_header(skb),
1302
								   fraggap, 0);
L
Linus Torvalds 已提交
1303 1304
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
1305
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
			}

			/*
			 * Put the packet on the pending queue.
			 */
			__skb_queue_tail(&sk->sk_write_queue, skb);
			continue;
		}

		if (len > size)
			len = size;
1317 1318

		if (skb_append_pagefrags(skb, page, offset, len)) {
L
Linus Torvalds 已提交
1319 1320 1321 1322 1323
			err = -EMSGSIZE;
			goto error;
		}

		if (skb->ip_summed == CHECKSUM_NONE) {
1324
			__wsum csum;
L
Linus Torvalds 已提交
1325 1326 1327 1328 1329 1330
			csum = csum_page(page, offset, len);
			skb->csum = csum_block_add(skb->csum, csum, skb->len);
		}

		skb->len += len;
		skb->data_len += len;
1331 1332
		skb->truesize += len;
		atomic_add(len, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1333 1334 1335 1336 1337 1338
		offset += len;
		size -= len;
	}
	return 0;

error:
1339
	cork->length -= size;
P
Pavel Emelyanov 已提交
1340
	IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1341 1342 1343
	return err;
}

1344
static void ip_cork_release(struct inet_cork *cork)
1345
{
1346 1347 1348 1349 1350
	cork->flags &= ~IPCORK_OPT;
	kfree(cork->opt);
	cork->opt = NULL;
	dst_release(cork->dst);
	cork->dst = NULL;
1351 1352
}

L
Linus Torvalds 已提交
1353 1354 1355 1356
/*
 *	Combined all pending IP fragments on the socket as one IP datagram
 *	and push them out.
 */
1357
struct sk_buff *__ip_make_skb(struct sock *sk,
1358
			      struct flowi4 *fl4,
1359 1360
			      struct sk_buff_head *queue,
			      struct inet_cork *cork)
L
Linus Torvalds 已提交
1361 1362 1363 1364
{
	struct sk_buff *skb, *tmp_skb;
	struct sk_buff **tail_skb;
	struct inet_sock *inet = inet_sk(sk);
1365
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
1366
	struct ip_options *opt = NULL;
1367
	struct rtable *rt = (struct rtable *)cork->dst;
L
Linus Torvalds 已提交
1368
	struct iphdr *iph;
1369
	__be16 df = 0;
L
Linus Torvalds 已提交
1370 1371
	__u8 ttl;

1372 1373
	skb = __skb_dequeue(queue);
	if (!skb)
L
Linus Torvalds 已提交
1374 1375 1376 1377
		goto out;
	tail_skb = &(skb_shinfo(skb)->frag_list);

	/* move skb->data to ip header from ext header */
1378
	if (skb->data < skb_network_header(skb))
1379
		__skb_pull(skb, skb_network_offset(skb));
1380
	while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1381
		__skb_pull(tmp_skb, skb_network_header_len(skb));
L
Linus Torvalds 已提交
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
		*tail_skb = tmp_skb;
		tail_skb = &(tmp_skb->next);
		skb->len += tmp_skb->len;
		skb->data_len += tmp_skb->len;
		skb->truesize += tmp_skb->truesize;
		tmp_skb->destructor = NULL;
		tmp_skb->sk = NULL;
	}

	/* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
	 * to fragment the frame generated here. No matter, what transforms
	 * how transforms change size of the packet, it will come out.
	 */
W
WANG Cong 已提交
1395
	skb->ignore_df = ip_sk_ignore_df(sk);
L
Linus Torvalds 已提交
1396 1397

	/* DF bit is set when we want to see DF on outgoing frames.
W
WANG Cong 已提交
1398
	 * If ignore_df is set too, we still allow to fragment this frame
L
Linus Torvalds 已提交
1399
	 * locally. */
1400 1401
	if (inet->pmtudisc == IP_PMTUDISC_DO ||
	    inet->pmtudisc == IP_PMTUDISC_PROBE ||
1402 1403
	    (skb->len <= dst_mtu(&rt->dst) &&
	     ip_dont_fragment(sk, &rt->dst)))
L
Linus Torvalds 已提交
1404 1405
		df = htons(IP_DF);

1406 1407
	if (cork->flags & IPCORK_OPT)
		opt = cork->opt;
L
Linus Torvalds 已提交
1408

1409 1410 1411
	if (cork->ttl != 0)
		ttl = cork->ttl;
	else if (rt->rt_type == RTN_MULTICAST)
L
Linus Torvalds 已提交
1412 1413
		ttl = inet->mc_ttl;
	else
1414
		ttl = ip_select_ttl(inet, &rt->dst);
L
Linus Torvalds 已提交
1415

1416
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
1417 1418
	iph->version = 4;
	iph->ihl = 5;
1419
	iph->tos = (cork->tos != -1) ? cork->tos : inet->tos;
L
Linus Torvalds 已提交
1420 1421 1422
	iph->frag_off = df;
	iph->ttl = ttl;
	iph->protocol = sk->sk_protocol;
1423
	ip_copy_addrs(iph, fl4);
1424
	ip_select_ident(net, skb, sk);
L
Linus Torvalds 已提交
1425

1426 1427 1428 1429 1430
	if (opt) {
		iph->ihl += opt->optlen>>2;
		ip_options_build(skb, opt, cork->addr, rt, 0);
	}

1431
	skb->priority = (cork->tos != -1) ? cork->priority: sk->sk_priority;
1432
	skb->mark = sk->sk_mark;
1433 1434 1435 1436
	/*
	 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
	 * on dst refcount
	 */
1437
	cork->dst = NULL;
1438
	skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
1439

1440
	if (iph->protocol == IPPROTO_ICMP)
1441
		icmp_out_count(net, ((struct icmphdr *)
1442 1443
			skb_transport_header(skb))->type);

1444 1445 1446 1447 1448
	ip_cork_release(cork);
out:
	return skb;
}

E
Eric Dumazet 已提交
1449
int ip_send_skb(struct net *net, struct sk_buff *skb)
1450 1451 1452
{
	int err;

1453
	err = ip_local_out(net, skb->sk, skb);
L
Linus Torvalds 已提交
1454 1455
	if (err) {
		if (err > 0)
E
Eric Dumazet 已提交
1456
			err = net_xmit_errno(err);
L
Linus Torvalds 已提交
1457
		if (err)
1458
			IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1459 1460 1461 1462 1463
	}

	return err;
}

1464
int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4)
1465
{
1466 1467
	struct sk_buff *skb;

1468
	skb = ip_finish_skb(sk, fl4);
1469 1470 1471 1472
	if (!skb)
		return 0;

	/* Netfilter gets whole the not fragmented skb. */
E
Eric Dumazet 已提交
1473
	return ip_send_skb(sock_net(sk), skb);
1474 1475
}

L
Linus Torvalds 已提交
1476 1477 1478
/*
 *	Throw away all pending data on the socket.
 */
1479 1480 1481
static void __ip_flush_pending_frames(struct sock *sk,
				      struct sk_buff_head *queue,
				      struct inet_cork *cork)
L
Linus Torvalds 已提交
1482 1483 1484
{
	struct sk_buff *skb;

1485
	while ((skb = __skb_dequeue_tail(queue)) != NULL)
L
Linus Torvalds 已提交
1486 1487
		kfree_skb(skb);

1488 1489 1490 1491 1492
	ip_cork_release(cork);
}

void ip_flush_pending_frames(struct sock *sk)
{
1493
	__ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
L
Linus Torvalds 已提交
1494 1495
}

1496
struct sk_buff *ip_make_skb(struct sock *sk,
1497
			    struct flowi4 *fl4,
1498 1499 1500 1501 1502 1503
			    int getfrag(void *from, char *to, int offset,
					int len, int odd, struct sk_buff *skb),
			    void *from, int length, int transhdrlen,
			    struct ipcm_cookie *ipc, struct rtable **rtp,
			    unsigned int flags)
{
1504
	struct inet_cork cork;
1505 1506 1507 1508 1509 1510 1511 1512
	struct sk_buff_head queue;
	int err;

	if (flags & MSG_PROBE)
		return NULL;

	__skb_queue_head_init(&queue);

1513 1514
	cork.flags = 0;
	cork.addr = 0;
1515
	cork.opt = NULL;
1516 1517 1518 1519
	err = ip_setup_cork(sk, &cork, ipc, rtp);
	if (err)
		return ERR_PTR(err);

1520 1521
	err = __ip_append_data(sk, fl4, &queue, &cork,
			       &current->task_frag, getfrag,
1522 1523 1524 1525 1526 1527
			       from, length, transhdrlen, flags);
	if (err) {
		__ip_flush_pending_frames(sk, &queue, &cork);
		return ERR_PTR(err);
	}

1528
	return __ip_make_skb(sk, fl4, &queue, &cork);
1529
}
L
Linus Torvalds 已提交
1530 1531 1532 1533

/*
 *	Fetch data from kernel space and fill in checksum if needed.
 */
1534
static int ip_reply_glue_bits(void *dptr, char *to, int offset,
L
Linus Torvalds 已提交
1535 1536
			      int len, int odd, struct sk_buff *skb)
{
1537
	__wsum csum;
L
Linus Torvalds 已提交
1538 1539 1540

	csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
	skb->csum = csum_block_add(skb->csum, csum, odd);
1541
	return 0;
L
Linus Torvalds 已提交
1542 1543
}

1544
/*
L
Linus Torvalds 已提交
1545
 *	Generic function to send a packet as reply to another packet.
1546
 *	Used to send some TCP resets/acks so far.
L
Linus Torvalds 已提交
1547
 */
1548
void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb,
1549 1550 1551
			   const struct ip_options *sopt,
			   __be32 daddr, __be32 saddr,
			   const struct ip_reply_arg *arg,
1552
			   unsigned int len)
L
Linus Torvalds 已提交
1553
{
1554
	struct ip_options_data replyopts;
L
Linus Torvalds 已提交
1555
	struct ipcm_cookie ipc;
1556
	struct flowi4 fl4;
E
Eric Dumazet 已提交
1557
	struct rtable *rt = skb_rtable(skb);
1558
	struct net *net = sock_net(sk);
1559
	struct sk_buff *nskb;
1560
	int err;
1561
	int oif;
L
Linus Torvalds 已提交
1562

1563
	if (__ip_options_echo(&replyopts.opt.opt, skb, sopt))
L
Linus Torvalds 已提交
1564 1565
		return;

1566
	ipc.addr = daddr;
L
Linus Torvalds 已提交
1567
	ipc.opt = NULL;
1568
	ipc.tx_flags = 0;
1569 1570
	ipc.ttl = 0;
	ipc.tos = -1;
L
Linus Torvalds 已提交
1571

1572
	if (replyopts.opt.opt.optlen) {
L
Linus Torvalds 已提交
1573 1574
		ipc.opt = &replyopts.opt;

1575 1576
		if (replyopts.opt.opt.srr)
			daddr = replyopts.opt.opt.faddr;
L
Linus Torvalds 已提交
1577 1578
	}

1579
	oif = arg->bound_dev_if;
1580
	oif = oif ? : skb->skb_iif;
1581 1582

	flowi4_init_output(&fl4, oif,
1583
			   IP4_REPLY_MARK(net, skb->mark),
1584
			   RT_TOS(arg->tos),
1585
			   RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol,
1586
			   ip_reply_arg_flowi_flags(arg),
1587
			   daddr, saddr,
1588 1589
			   tcp_hdr(skb)->source, tcp_hdr(skb)->dest);
	security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
1590
	rt = ip_route_output_key(net, &fl4);
1591 1592
	if (IS_ERR(rt))
		return;
L
Linus Torvalds 已提交
1593

1594
	inet_sk(sk)->tos = arg->tos;
L
Linus Torvalds 已提交
1595 1596

	sk->sk_priority = skb->priority;
1597
	sk->sk_protocol = ip_hdr(skb)->protocol;
1598
	sk->sk_bound_dev_if = arg->bound_dev_if;
1599
	sk->sk_sndbuf = sysctl_wmem_default;
1600 1601 1602 1603 1604 1605 1606
	err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base,
			     len, 0, &ipc, &rt, MSG_DONTWAIT);
	if (unlikely(err)) {
		ip_flush_pending_frames(sk);
		goto out;
	}

1607 1608
	nskb = skb_peek(&sk->sk_write_queue);
	if (nskb) {
L
Linus Torvalds 已提交
1609
		if (arg->csumoffset >= 0)
1610 1611
			*((__sum16 *)skb_transport_header(nskb) +
			  arg->csumoffset) = csum_fold(csum_add(nskb->csum,
1612
								arg->csum));
1613
		nskb->ip_summed = CHECKSUM_NONE;
1614
		ip_push_pending_frames(sk, &fl4);
L
Linus Torvalds 已提交
1615
	}
1616
out:
L
Linus Torvalds 已提交
1617 1618 1619 1620 1621 1622 1623 1624
	ip_rt_put(rt);
}

void __init ip_init(void)
{
	ip_rt_init();
	inet_initpeers();

1625 1626
#if defined(CONFIG_IP_MULTICAST)
	igmp_mc_init();
L
Linus Torvalds 已提交
1627 1628
#endif
}