ip_output.c 39.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		The Internet Protocol (IP) output module.
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Donald Becker, <becker@super.org>
 *		Alan Cox, <Alan.Cox@linux.org>
 *		Richard Underwood
 *		Stefan Becker, <stefanb@yello.ping.de>
 *		Jorge Cwik, <jorge@laser.satlink.net>
 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *		Hirokazu Takahashi, <taka@valinux.co.jp>
 *
 *	See ip_input.c for original log
 *
 *	Fixes:
 *		Alan Cox	:	Missing nonblock feature in ip_build_xmit.
 *		Mike Kilburn	:	htons() missing in ip_build_xmit.
23
 *		Bradford Johnson:	Fix faulty handling of some frames when
L
Linus Torvalds 已提交
24 25 26 27 28 29 30 31 32 33
 *					no route is found.
 *		Alexander Demenshin:	Missing sk/skb free in ip_queue_xmit
 *					(in case if packet not accepted by
 *					output firewall rules)
 *		Mike McLagan	:	Routing by source
 *		Alexey Kuznetsov:	use new route cache
 *		Andi Kleen:		Fix broken PMTU recovery and remove
 *					some redundant tests.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
 *		Andi Kleen	: 	Replace ip_reply with ip_send_reply.
34 35 36
 *		Andi Kleen	:	Split fast and slow ip_build_xmit path
 *					for decreased register pressure on x86
 *					and more readibility.
L
Linus Torvalds 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
 *		Marc Boucher	:	When call_out_firewall returns FW_QUEUE,
 *					silently drop skb instead of failing with -EPERM.
 *		Detlev Wengorz	:	Copy protocol for fragments.
 *		Hirokazu Takahashi:	HW checksumming for outgoing UDP
 *					datagrams.
 *		Hirokazu Takahashi:	sendfile() on UDP works now.
 */

#include <asm/uaccess.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/errno.h>
A
Al Viro 已提交
52
#include <linux/highmem.h>
53
#include <linux/slab.h>
L
Linus Torvalds 已提交
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68

#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/init.h>

#include <net/snmp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
69
#include <net/xfrm.h>
L
Linus Torvalds 已提交
70 71 72 73 74 75 76 77 78 79 80
#include <linux/skbuff.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/icmp.h>
#include <net/checksum.h>
#include <net/inetpeer.h>
#include <linux/igmp.h>
#include <linux/netfilter_ipv4.h>
#include <linux/netfilter_bridge.h>
#include <linux/mroute.h>
#include <linux/netlink.h>
81
#include <linux/tcp.h>
L
Linus Torvalds 已提交
82

83
int sysctl_ip_default_ttl __read_mostly = IPDEFTTL;
84
EXPORT_SYMBOL(sysctl_ip_default_ttl);
L
Linus Torvalds 已提交
85

86 87 88 89
static int
ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
	    unsigned int mtu,
	    int (*output)(struct net *, struct sock *, struct sk_buff *));
90

L
Linus Torvalds 已提交
91
/* Generate a checksum for an outgoing IP datagram. */
92
void ip_send_check(struct iphdr *iph)
L
Linus Torvalds 已提交
93 94 95 96
{
	iph->check = 0;
	iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
}
E
Eric Dumazet 已提交
97
EXPORT_SYMBOL(ip_send_check);
L
Linus Torvalds 已提交
98

99
int __ip_local_out(struct sock *sk, struct sk_buff *skb)
H
Herbert Xu 已提交
100
{
101
	struct net *net = dev_net(skb_dst(skb)->dev);
H
Herbert Xu 已提交
102 103 104 105
	struct iphdr *iph = ip_hdr(skb);

	iph->tot_len = htons(skb->len);
	ip_send_check(iph);
106 107
	return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT,
		       net, sk, skb, NULL, skb_dst(skb)->dev,
108
		       dst_output);
109 110
}

111
int ip_local_out_sk(struct sock *sk, struct sk_buff *skb)
H
Herbert Xu 已提交
112
{
113
	struct net *net = dev_net(skb_dst(skb)->dev);
H
Herbert Xu 已提交
114 115
	int err;

116
	err = __ip_local_out(sk, skb);
H
Herbert Xu 已提交
117
	if (likely(err == 1))
118
		err = dst_output(net, sk, skb);
H
Herbert Xu 已提交
119 120 121

	return err;
}
122
EXPORT_SYMBOL_GPL(ip_local_out_sk);
H
Herbert Xu 已提交
123

L
Linus Torvalds 已提交
124 125 126 127 128
static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
{
	int ttl = inet->uc_ttl;

	if (ttl < 0)
129
		ttl = ip4_dst_hoplimit(dst);
L
Linus Torvalds 已提交
130 131 132
	return ttl;
}

133
/*
L
Linus Torvalds 已提交
134 135 136
 *		Add an ip header to a skbuff and send it out.
 *
 */
137
int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk,
138
			  __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)
L
Linus Torvalds 已提交
139 140
{
	struct inet_sock *inet = inet_sk(sk);
E
Eric Dumazet 已提交
141
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
142 143 144
	struct iphdr *iph;

	/* Build the IP header. */
145
	skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
146
	skb_reset_network_header(skb);
147
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
148 149 150
	iph->version  = 4;
	iph->ihl      = 5;
	iph->tos      = inet->tos;
151
	iph->ttl      = ip_select_ttl(inet, &rt->dst);
152 153
	iph->daddr    = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
	iph->saddr    = saddr;
L
Linus Torvalds 已提交
154
	iph->protocol = sk->sk_protocol;
155 156 157 158 159 160 161
	if (ip_dont_fragment(sk, &rt->dst)) {
		iph->frag_off = htons(IP_DF);
		iph->id = 0;
	} else {
		iph->frag_off = 0;
		__ip_select_ident(sock_net(sk), iph, 1);
	}
L
Linus Torvalds 已提交
162

163 164 165
	if (opt && opt->opt.optlen) {
		iph->ihl += opt->opt.optlen>>2;
		ip_options_build(skb, &opt->opt, daddr, rt, 0);
L
Linus Torvalds 已提交
166 167 168
	}

	skb->priority = sk->sk_priority;
169
	skb->mark = sk->sk_mark;
L
Linus Torvalds 已提交
170 171

	/* Send it out. */
H
Herbert Xu 已提交
172
	return ip_local_out(skb);
L
Linus Torvalds 已提交
173
}
174 175
EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);

176
static int ip_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
177
{
E
Eric Dumazet 已提交
178
	struct dst_entry *dst = skb_dst(skb);
179
	struct rtable *rt = (struct rtable *)dst;
L
Linus Torvalds 已提交
180
	struct net_device *dev = dst->dev;
181
	unsigned int hh_len = LL_RESERVED_SPACE(dev);
182
	struct neighbour *neigh;
183
	u32 nexthop;
L
Linus Torvalds 已提交
184

185
	if (rt->rt_type == RTN_MULTICAST) {
186
		IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTMCAST, skb->len);
187
	} else if (rt->rt_type == RTN_BROADCAST)
188
		IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTBCAST, skb->len);
189

L
Linus Torvalds 已提交
190
	/* Be paranoid, rather than too clever. */
191
	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
L
Linus Torvalds 已提交
192 193 194
		struct sk_buff *skb2;

		skb2 = skb_realloc_headroom(skb, LL_RESERVED_SPACE(dev));
195
		if (!skb2) {
L
Linus Torvalds 已提交
196 197 198 199 200
			kfree_skb(skb);
			return -ENOMEM;
		}
		if (skb->sk)
			skb_set_owner_w(skb2, skb->sk);
201
		consume_skb(skb);
L
Linus Torvalds 已提交
202 203 204
		skb = skb2;
	}

205
	rcu_read_lock_bh();
J
Julian Anastasov 已提交
206
	nexthop = (__force u32) rt_nexthop(rt, ip_hdr(skb)->daddr);
207 208 209
	neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
	if (unlikely(!neigh))
		neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
210
	if (!IS_ERR(neigh)) {
211
		int res = dst_neigh_output(dst, neigh, skb);
212

213
		rcu_read_unlock_bh();
214 215
		return res;
	}
216
	rcu_read_unlock_bh();
217

218 219
	net_dbg_ratelimited("%s: No header cache and no neighbour!\n",
			    __func__);
L
Linus Torvalds 已提交
220 221 222 223
	kfree_skb(skb);
	return -EINVAL;
}

224 225
static int ip_finish_output_gso(struct net *net, struct sock *sk,
				struct sk_buff *skb, unsigned int mtu)
226 227 228 229 230 231 232
{
	netdev_features_t features;
	struct sk_buff *segs;
	int ret = 0;

	/* common case: locally created skb or seglen is <= mtu */
	if (((IPCB(skb)->flags & IPSKB_FORWARDED) == 0) ||
233
	      skb_gso_network_seglen(skb) <= mtu)
234
		return ip_finish_output2(net, sk, skb);
235 236 237 238 239 240 241 242 243 244

	/* Slowpath -  GSO segment length is exceeding the dst MTU.
	 *
	 * This can happen in two cases:
	 * 1) TCP GRO packet, DF bit not set
	 * 2) skb arrived via virtio-net, we thus get TSO/GSO skbs directly
	 * from host network stack.
	 */
	features = netif_skb_features(skb);
	segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
245
	if (IS_ERR_OR_NULL(segs)) {
246 247 248 249 250 251 252 253 254 255 256
		kfree_skb(skb);
		return -ENOMEM;
	}

	consume_skb(skb);

	do {
		struct sk_buff *nskb = segs->next;
		int err;

		segs->next = NULL;
257
		err = ip_fragment(net, sk, segs, mtu, ip_finish_output2);
258 259 260 261 262 263 264 265 266

		if (err && ret == 0)
			ret = err;
		segs = nskb;
	} while (segs);

	return ret;
}

267
static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
268
{
269 270
	unsigned int mtu;

271 272
#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
	/* Policy lookup after SNAT yielded a new policy */
273
	if (skb_dst(skb)->xfrm) {
274
		IPCB(skb)->flags |= IPSKB_REROUTED;
275
		return dst_output(net, sk, skb);
276
	}
277
#endif
278
	mtu = ip_skb_dst_mtu(skb);
279
	if (skb_is_gso(skb))
280
		return ip_finish_output_gso(net, sk, skb, mtu);
281

282
	if (skb->len > mtu || (IPCB(skb)->flags & IPSKB_FRAG_PMTU))
283
		return ip_fragment(net, sk, skb, mtu, ip_finish_output2);
284

285
	return ip_finish_output2(net, sk, skb);
L
Linus Torvalds 已提交
286 287
}

288
int ip_mc_output(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
289
{
E
Eric Dumazet 已提交
290
	struct rtable *rt = skb_rtable(skb);
291
	struct net_device *dev = rt->dst.dev;
292
	struct net *net = dev_net(dev);
L
Linus Torvalds 已提交
293 294 295 296

	/*
	 *	If the indicated interface is up and running, send the packet.
	 */
297
	IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
L
Linus Torvalds 已提交
298 299 300 301 302 303 304 305 306

	skb->dev = dev;
	skb->protocol = htons(ETH_P_IP);

	/*
	 *	Multicasts are looped back for other local users
	 */

	if (rt->rt_flags&RTCF_MULTICAST) {
307
		if (sk_mc_loop(sk)
L
Linus Torvalds 已提交
308 309 310 311 312 313 314 315 316
#ifdef CONFIG_IP_MROUTE
		/* Small optimization: do not loopback not local frames,
		   which returned after forwarding; they will be  dropped
		   by ip_mr_input in any case.
		   Note, that local frames are looped back to be delivered
		   to local recipients.

		   This check is duplicated in ip_mr_input at the moment.
		 */
317 318 319
		    &&
		    ((rt->rt_flags & RTCF_LOCAL) ||
		     !(IPCB(skb)->flags & IPSKB_FORWARDED))
L
Linus Torvalds 已提交
320
#endif
321
		   ) {
L
Linus Torvalds 已提交
322 323
			struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
			if (newskb)
324
				NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
325
					net, sk, newskb, NULL, newskb->dev,
326
					dev_loopback_xmit);
L
Linus Torvalds 已提交
327 328 329 330
		}

		/* Multicasts with ttl 0 must not go beyond the host */

331
		if (ip_hdr(skb)->ttl == 0) {
L
Linus Torvalds 已提交
332 333 334 335 336 337 338 339
			kfree_skb(skb);
			return 0;
		}
	}

	if (rt->rt_flags&RTCF_BROADCAST) {
		struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
		if (newskb)
340 341 342
			NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
				net, sk, newskb, NULL, newskb->dev,
				dev_loopback_xmit);
L
Linus Torvalds 已提交
343 344
	}

345 346 347
	return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
			    net, sk, skb, NULL, skb->dev,
			    ip_finish_output,
348
			    !(IPCB(skb)->flags & IPSKB_REROUTED));
L
Linus Torvalds 已提交
349 350
}

351
int ip_output(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
352
{
E
Eric Dumazet 已提交
353
	struct net_device *dev = skb_dst(skb)->dev;
354
	struct net *net = dev_net(dev);
355

356
	IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
L
Linus Torvalds 已提交
357

358 359 360
	skb->dev = dev;
	skb->protocol = htons(ETH_P_IP);

361 362
	return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
			    net, sk, skb, NULL, dev,
363
			    ip_finish_output,
364
			    !(IPCB(skb)->flags & IPSKB_REROUTED));
L
Linus Torvalds 已提交
365 366
}

367 368 369 370 371 372 373 374 375 376 377 378 379 380
/*
 * copy saddr and daddr, possibly using 64bit load/stores
 * Equivalent to :
 *   iph->saddr = fl4->saddr;
 *   iph->daddr = fl4->daddr;
 */
static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4)
{
	BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) !=
		     offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr));
	memcpy(&iph->saddr, &fl4->saddr,
	       sizeof(fl4->saddr) + sizeof(fl4->daddr));
}

381 382
/* Note: skb->sk can be different from sk, in case of tunnels */
int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
L
Linus Torvalds 已提交
383 384
{
	struct inet_sock *inet = inet_sk(sk);
385
	struct ip_options_rcu *inet_opt;
386
	struct flowi4 *fl4;
L
Linus Torvalds 已提交
387 388
	struct rtable *rt;
	struct iphdr *iph;
389
	int res;
L
Linus Torvalds 已提交
390 391 392 393

	/* Skip all of this if the packet is already routed,
	 * f.e. by something like SCTP.
	 */
394
	rcu_read_lock();
395
	inet_opt = rcu_dereference(inet->inet_opt);
396
	fl4 = &fl->u.ip4;
E
Eric Dumazet 已提交
397
	rt = skb_rtable(skb);
398
	if (rt)
L
Linus Torvalds 已提交
399 400 401 402
		goto packet_routed;

	/* Make sure we can route this packet. */
	rt = (struct rtable *)__sk_dst_check(sk, 0);
403
	if (!rt) {
A
Al Viro 已提交
404
		__be32 daddr;
L
Linus Torvalds 已提交
405 406

		/* Use correct destination address if we have options. */
E
Eric Dumazet 已提交
407
		daddr = inet->inet_daddr;
408 409
		if (inet_opt && inet_opt->opt.srr)
			daddr = inet_opt->opt.faddr;
L
Linus Torvalds 已提交
410

411 412 413 414
		/* If this fails, retransmit mechanism of transport layer will
		 * keep trying until route appears or the connection times
		 * itself out.
		 */
415
		rt = ip_route_output_ports(sock_net(sk), fl4, sk,
416 417 418 419 420 421 422 423
					   daddr, inet->inet_saddr,
					   inet->inet_dport,
					   inet->inet_sport,
					   sk->sk_protocol,
					   RT_CONN_FLAGS(sk),
					   sk->sk_bound_dev_if);
		if (IS_ERR(rt))
			goto no_route;
424
		sk_setup_caps(sk, &rt->dst);
L
Linus Torvalds 已提交
425
	}
426
	skb_dst_set_noref(skb, &rt->dst);
L
Linus Torvalds 已提交
427 428

packet_routed:
J
Julian Anastasov 已提交
429
	if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway)
L
Linus Torvalds 已提交
430 431 432
		goto no_route;

	/* OK, we know where to send it, allocate and build IP header. */
433
	skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
434
	skb_reset_network_header(skb);
435
	iph = ip_hdr(skb);
436
	*((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (inet->tos & 0xff));
W
WANG Cong 已提交
437
	if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df)
L
Linus Torvalds 已提交
438 439 440
		iph->frag_off = htons(IP_DF);
	else
		iph->frag_off = 0;
441
	iph->ttl      = ip_select_ttl(inet, &rt->dst);
L
Linus Torvalds 已提交
442
	iph->protocol = sk->sk_protocol;
443 444
	ip_copy_addrs(iph, fl4);

L
Linus Torvalds 已提交
445 446
	/* Transport layer set skb->h.foo itself. */

447 448 449
	if (inet_opt && inet_opt->opt.optlen) {
		iph->ihl += inet_opt->opt.optlen >> 2;
		ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, 0);
L
Linus Torvalds 已提交
450 451
	}

452 453
	ip_select_ident_segs(sock_net(sk), skb, sk,
			     skb_shinfo(skb)->gso_segs ?: 1);
L
Linus Torvalds 已提交
454

455
	/* TODO : should we use skb->sk here instead of sk ? */
L
Linus Torvalds 已提交
456
	skb->priority = sk->sk_priority;
457
	skb->mark = sk->sk_mark;
L
Linus Torvalds 已提交
458

459
	res = ip_local_out_sk(sk, skb);
460 461
	rcu_read_unlock();
	return res;
L
Linus Torvalds 已提交
462 463

no_route:
464
	rcu_read_unlock();
P
Pavel Emelyanov 已提交
465
	IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
L
Linus Torvalds 已提交
466 467 468
	kfree_skb(skb);
	return -EHOSTUNREACH;
}
E
Eric Dumazet 已提交
469
EXPORT_SYMBOL(ip_queue_xmit);
L
Linus Torvalds 已提交
470 471 472 473 474 475

static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
{
	to->pkt_type = from->pkt_type;
	to->priority = from->priority;
	to->protocol = from->protocol;
E
Eric Dumazet 已提交
476
	skb_dst_drop(to);
477
	skb_dst_copy(to, from);
L
Linus Torvalds 已提交
478
	to->dev = from->dev;
T
Thomas Graf 已提交
479
	to->mark = from->mark;
L
Linus Torvalds 已提交
480 481 482 483 484 485 486

	/* Copy the flags to each fragment. */
	IPCB(to)->flags = IPCB(from)->flags;

#ifdef CONFIG_NET_SCHED
	to->tc_index = from->tc_index;
#endif
487
	nf_copy(to, from);
488 489
#if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
	to->ipvs_property = from->ipvs_property;
L
Linus Torvalds 已提交
490
#endif
491
	skb_copy_secmark(to, from);
L
Linus Torvalds 已提交
492 493
}

494
static int ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
495
		       unsigned int mtu,
496
		       int (*output)(struct net *, struct sock *, struct sk_buff *))
497 498 499
{
	struct iphdr *iph = ip_hdr(skb);

500
	if ((iph->frag_off & htons(IP_DF)) == 0)
501
		return ip_do_fragment(net, sk, skb, output);
502 503

	if (unlikely(!skb->ignore_df ||
504 505
		     (IPCB(skb)->frag_max_size &&
		      IPCB(skb)->frag_max_size > mtu))) {
506
		IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
507 508 509 510 511 512
		icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
			  htonl(mtu));
		kfree_skb(skb);
		return -EMSGSIZE;
	}

513
	return ip_do_fragment(net, sk, skb, output);
514 515
}

L
Linus Torvalds 已提交
516 517 518 519 520 521 522
/*
 *	This IP datagram is too large to be sent in one piece.  Break it up into
 *	smaller pieces (each of size equal to IP header plus
 *	a block of the data of the original IP data part) that will yet fit in a
 *	single device frame, and queue such a frame for sending.
 */

523 524
int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
		   int (*output)(struct net *, struct sock *, struct sk_buff *))
L
Linus Torvalds 已提交
525 526 527 528 529
{
	struct iphdr *iph;
	int ptr;
	struct net_device *dev;
	struct sk_buff *skb2;
530
	unsigned int mtu, hlen, left, len, ll_rs;
L
Linus Torvalds 已提交
531
	int offset;
532
	__be16 not_last_frag;
E
Eric Dumazet 已提交
533
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
534 535
	int err = 0;

536
	dev = rt->dst.dev;
L
Linus Torvalds 已提交
537 538 539 540 541

	/*
	 *	Point into the IP datagram header.
	 */

542
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
543

544
	mtu = ip_skb_dst_mtu(skb);
545 546
	if (IPCB(skb)->frag_max_size && IPCB(skb)->frag_max_size < mtu)
		mtu = IPCB(skb)->frag_max_size;
L
Linus Torvalds 已提交
547 548 549 550 551 552

	/*
	 *	Setup starting values.
	 */

	hlen = iph->ihl * 4;
553
	mtu = mtu - hlen;	/* Size of data space */
H
Herbert Xu 已提交
554
	IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
L
Linus Torvalds 已提交
555 556 557 558 559 560 561 562

	/* When frag_list is given, use it. First, check its validity:
	 * some transformers could create wrong frag_list or break existing
	 * one, it is not prohibited. In this case fall back to copying.
	 *
	 * LATER: this step can be merged to real generation of fragments,
	 * we can switch to copy when see the first bad fragment.
	 */
563
	if (skb_has_frag_list(skb)) {
564
		struct sk_buff *frag, *frag2;
L
Linus Torvalds 已提交
565 566 567 568
		int first_len = skb_pagelen(skb);

		if (first_len - hlen > mtu ||
		    ((first_len - hlen) & 7) ||
569
		    ip_is_fragment(iph) ||
L
Linus Torvalds 已提交
570 571 572
		    skb_cloned(skb))
			goto slow_path;

573
		skb_walk_frags(skb, frag) {
L
Linus Torvalds 已提交
574 575 576 577
			/* Correct geometry. */
			if (frag->len > mtu ||
			    ((frag->len & 7) && frag->next) ||
			    skb_headroom(frag) < hlen)
578
				goto slow_path_clean;
L
Linus Torvalds 已提交
579 580 581

			/* Partially cloned skb? */
			if (skb_shared(frag))
582
				goto slow_path_clean;
583 584 585 586 587 588

			BUG_ON(frag->sk);
			if (skb->sk) {
				frag->sk = skb->sk;
				frag->destructor = sock_wfree;
			}
589
			skb->truesize -= frag->truesize;
L
Linus Torvalds 已提交
590 591 592 593 594 595 596
		}

		/* Everything is OK. Generate! */

		err = 0;
		offset = 0;
		frag = skb_shinfo(skb)->frag_list;
597
		skb_frag_list_init(skb);
L
Linus Torvalds 已提交
598 599 600 601 602 603 604 605 606 607 608
		skb->data_len = first_len - skb_headlen(skb);
		skb->len = first_len;
		iph->tot_len = htons(first_len);
		iph->frag_off = htons(IP_MF);
		ip_send_check(iph);

		for (;;) {
			/* Prepare header of the next frame,
			 * before previous one went down. */
			if (frag) {
				frag->ip_summed = CHECKSUM_NONE;
609
				skb_reset_transport_header(frag);
610 611
				__skb_push(frag, hlen);
				skb_reset_network_header(frag);
612
				memcpy(skb_network_header(frag), iph, hlen);
613
				iph = ip_hdr(frag);
L
Linus Torvalds 已提交
614 615 616 617 618 619
				iph->tot_len = htons(frag->len);
				ip_copy_metadata(frag, skb);
				if (offset == 0)
					ip_options_fragment(frag);
				offset += skb->len - hlen;
				iph->frag_off = htons(offset>>3);
620
				if (frag->next)
L
Linus Torvalds 已提交
621 622 623 624 625
					iph->frag_off |= htons(IP_MF);
				/* Ready, complete checksum */
				ip_send_check(iph);
			}

626
			err = output(net, sk, skb);
L
Linus Torvalds 已提交
627

628
			if (!err)
629
				IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
630 631 632 633 634 635 636 637 638
			if (err || !frag)
				break;

			skb = frag;
			frag = skb->next;
			skb->next = NULL;
		}

		if (err == 0) {
639
			IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
L
Linus Torvalds 已提交
640 641 642 643 644 645 646 647
			return 0;
		}

		while (frag) {
			skb = frag->next;
			kfree_skb(frag);
			frag = skb;
		}
648
		IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
649
		return err;
650 651 652 653 654 655 656 657 658

slow_path_clean:
		skb_walk_frags(skb, frag2) {
			if (frag2 == frag)
				break;
			frag2->sk = NULL;
			frag2->destructor = NULL;
			skb->truesize += frag2->truesize;
		}
L
Linus Torvalds 已提交
659 660 661
	}

slow_path:
662 663 664
	/* for offloaded checksums cleanup checksum before fragmentation */
	if ((skb->ip_summed == CHECKSUM_PARTIAL) && skb_checksum_help(skb))
		goto fail;
665
	iph = ip_hdr(skb);
666

L
Linus Torvalds 已提交
667
	left = skb->len - hlen;		/* Space per frame */
668
	ptr = hlen;		/* Where to start from */
L
Linus Torvalds 已提交
669

670
	ll_rs = LL_RESERVED_SPACE(rt->dst.dev);
671

L
Linus Torvalds 已提交
672 673 674 675 676 677 678 679 680 681 682
	/*
	 *	Fragment the datagram.
	 */

	offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
	not_last_frag = iph->frag_off & htons(IP_MF);

	/*
	 *	Keep copying data until we run out.
	 */

S
Stephen Hemminger 已提交
683
	while (left > 0) {
L
Linus Torvalds 已提交
684 685 686 687
		len = left;
		/* IF: it doesn't fit, use 'mtu' - the data space left */
		if (len > mtu)
			len = mtu;
L
Lucas De Marchi 已提交
688
		/* IF: we are not sending up to and including the packet end
L
Linus Torvalds 已提交
689 690 691 692 693
		   then align the next start on an eight byte boundary */
		if (len < left)	{
			len &= ~7;
		}

694 695 696
		/* Allocate buffer */
		skb2 = alloc_skb(len + hlen + ll_rs, GFP_ATOMIC);
		if (!skb2) {
L
Linus Torvalds 已提交
697 698 699 700 701 702 703 704 705 706 707
			err = -ENOMEM;
			goto fail;
		}

		/*
		 *	Set up data on packet
		 */

		ip_copy_metadata(skb2, skb);
		skb_reserve(skb2, ll_rs);
		skb_put(skb2, len + hlen);
708
		skb_reset_network_header(skb2);
709
		skb2->transport_header = skb2->network_header + hlen;
L
Linus Torvalds 已提交
710 711 712 713 714 715 716 717 718 719 720 721 722

		/*
		 *	Charge the memory for the fragment to any owner
		 *	it might possess
		 */

		if (skb->sk)
			skb_set_owner_w(skb2, skb->sk);

		/*
		 *	Copy the packet header into the new buffer.
		 */

723
		skb_copy_from_linear_data(skb, skb_network_header(skb2), hlen);
L
Linus Torvalds 已提交
724 725 726 727

		/*
		 *	Copy a block of the IP datagram.
		 */
728
		if (skb_copy_bits(skb, ptr, skb_transport_header(skb2), len))
L
Linus Torvalds 已提交
729 730 731 732 733 734
			BUG();
		left -= len;

		/*
		 *	Fill in the new header fields.
		 */
735
		iph = ip_hdr(skb2);
L
Linus Torvalds 已提交
736 737
		iph->frag_off = htons((offset >> 3));

738 739 740
		if (IPCB(skb)->flags & IPSKB_FRAG_PMTU)
			iph->frag_off |= htons(IP_DF);

L
Linus Torvalds 已提交
741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765
		/* ANK: dirty, but effective trick. Upgrade options only if
		 * the segment to be fragmented was THE FIRST (otherwise,
		 * options are already fixed) and make it ONCE
		 * on the initial skb, so that all the following fragments
		 * will inherit fixed options.
		 */
		if (offset == 0)
			ip_options_fragment(skb);

		/*
		 *	Added AC : If we are fragmenting a fragment that's not the
		 *		   last fragment then keep MF on each bit
		 */
		if (left > 0 || not_last_frag)
			iph->frag_off |= htons(IP_MF);
		ptr += len;
		offset += len;

		/*
		 *	Put this fragment into the sending queue.
		 */
		iph->tot_len = htons(len + hlen);

		ip_send_check(iph);

766
		err = output(net, sk, skb2);
L
Linus Torvalds 已提交
767 768
		if (err)
			goto fail;
769

770
		IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
L
Linus Torvalds 已提交
771
	}
772
	consume_skb(skb);
773
	IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
L
Linus Torvalds 已提交
774 775 776
	return err;

fail:
777
	kfree_skb(skb);
778
	IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
L
Linus Torvalds 已提交
779 780
	return err;
}
781
EXPORT_SYMBOL(ip_do_fragment);
782

L
Linus Torvalds 已提交
783 784 785
int
ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
{
786
	struct msghdr *msg = from;
L
Linus Torvalds 已提交
787

788
	if (skb->ip_summed == CHECKSUM_PARTIAL) {
789
		if (copy_from_iter(to, len, &msg->msg_iter) != len)
L
Linus Torvalds 已提交
790 791
			return -EFAULT;
	} else {
792
		__wsum csum = 0;
793
		if (csum_and_copy_from_iter(to, len, &csum, &msg->msg_iter) != len)
L
Linus Torvalds 已提交
794 795 796 797 798
			return -EFAULT;
		skb->csum = csum_block_add(skb->csum, csum, odd);
	}
	return 0;
}
E
Eric Dumazet 已提交
799
EXPORT_SYMBOL(ip_generic_getfrag);
L
Linus Torvalds 已提交
800

801
static inline __wsum
L
Linus Torvalds 已提交
802 803 804
csum_page(struct page *page, int offset, int copy)
{
	char *kaddr;
805
	__wsum csum;
L
Linus Torvalds 已提交
806 807 808 809 810 811
	kaddr = kmap(page);
	csum = csum_partial(kaddr + offset, copy, 0);
	kunmap(page);
	return csum;
}

A
Adrian Bunk 已提交
812
static inline int ip_ufo_append_data(struct sock *sk,
813
			struct sk_buff_head *queue,
814 815 816
			int getfrag(void *from, char *to, int offset, int len,
			       int odd, struct sk_buff *skb),
			void *from, int length, int hh_len, int fragheaderlen,
817
			int transhdrlen, int maxfraglen, unsigned int flags)
818 819 820 821 822 823 824 825
{
	struct sk_buff *skb;
	int err;

	/* There is support for UDP fragmentation offload by network
	 * device, so create one single skb packet containing complete
	 * udp datagram
	 */
826 827
	skb = skb_peek_tail(queue);
	if (!skb) {
828 829 830 831
		skb = sock_alloc_send_skb(sk,
			hh_len + fragheaderlen + transhdrlen + 20,
			(flags & MSG_DONTWAIT), &err);

832
		if (!skb)
833 834 835 836 837 838
			return err;

		/* reserve space for Hardware header */
		skb_reserve(skb, hh_len);

		/* create space for UDP/IP header */
839
		skb_put(skb, fragheaderlen + transhdrlen);
840 841

		/* initialize network header pointer */
842
		skb_reset_network_header(skb);
843 844

		/* initialize protocol header pointer */
845
		skb->transport_header = skb->network_header + fragheaderlen;
846 847 848

		skb->csum = 0;

849
		__skb_queue_tail(queue, skb);
850 851
	} else if (skb_is_gso(skb)) {
		goto append;
852
	}
853

854 855 856 857 858 859
	skb->ip_summed = CHECKSUM_PARTIAL;
	/* specify the length of each IP datagram fragment */
	skb_shinfo(skb)->gso_size = maxfraglen - fragheaderlen;
	skb_shinfo(skb)->gso_type = SKB_GSO_UDP;

append:
860 861
	return skb_append_datato_frags(sk, skb, getfrag, from,
				       (length - transhdrlen));
862 863
}

864 865 866
static int __ip_append_data(struct sock *sk,
			    struct flowi4 *fl4,
			    struct sk_buff_head *queue,
867
			    struct inet_cork *cork,
868
			    struct page_frag *pfrag,
869 870 871 872
			    int getfrag(void *from, char *to, int offset,
					int len, int odd, struct sk_buff *skb),
			    void *from, int length, int transhdrlen,
			    unsigned int flags)
L
Linus Torvalds 已提交
873 874 875 876
{
	struct inet_sock *inet = inet_sk(sk);
	struct sk_buff *skb;

877
	struct ip_options *opt = cork->opt;
L
Linus Torvalds 已提交
878 879 880 881 882 883
	int hh_len;
	int exthdrlen;
	int mtu;
	int copy;
	int err;
	int offset = 0;
884
	unsigned int maxfraglen, fragheaderlen, maxnonfragsize;
L
Linus Torvalds 已提交
885
	int csummode = CHECKSUM_NONE;
886
	struct rtable *rt = (struct rtable *)cork->dst;
887
	u32 tskey = 0;
L
Linus Torvalds 已提交
888

889 890 891
	skb = skb_peek_tail(queue);

	exthdrlen = !skb ? rt->dst.header_len : 0;
892
	mtu = cork->fragsize;
893 894 895
	if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&
	    sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
		tskey = sk->sk_tskey++;
L
Linus Torvalds 已提交
896

897
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
L
Linus Torvalds 已提交
898 899 900

	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
W
WANG Cong 已提交
901
	maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu;
L
Linus Torvalds 已提交
902

903
	if (cork->length + length > maxnonfragsize - fragheaderlen) {
904
		ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
905
			       mtu - (opt ? opt->optlen : 0));
L
Linus Torvalds 已提交
906 907 908 909 910 911 912 913 914
		return -EMSGSIZE;
	}

	/*
	 * transhdrlen > 0 means that this is the first fragment and we wish
	 * it won't be fragmented in the future.
	 */
	if (transhdrlen &&
	    length + fragheaderlen <= mtu &&
915
	    rt->dst.dev->features & NETIF_F_V4_CSUM &&
L
Linus Torvalds 已提交
916
	    !exthdrlen)
917
		csummode = CHECKSUM_PARTIAL;
L
Linus Torvalds 已提交
918

919
	cork->length += length;
920
	if (((length > mtu) || (skb && skb_is_gso(skb))) &&
921
	    (sk->sk_protocol == IPPROTO_UDP) &&
922 923
	    (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len &&
	    (sk->sk_type == SOCK_DGRAM)) {
924 925
		err = ip_ufo_append_data(sk, queue, getfrag, from, length,
					 hh_len, fragheaderlen, transhdrlen,
926
					 maxfraglen, flags);
927
		if (err)
928 929 930
			goto error;
		return 0;
	}
L
Linus Torvalds 已提交
931 932 933 934 935 936 937 938

	/* So, what's going on in the loop below?
	 *
	 * We use calculated fragment length to generate chained skb,
	 * each of segments is IP fragment ready for sending to network after
	 * adding appropriate IP header.
	 */

939
	if (!skb)
L
Linus Torvalds 已提交
940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969
		goto alloc_new_skb;

	while (length > 0) {
		/* Check if the remaining data fits into current packet. */
		copy = mtu - skb->len;
		if (copy < length)
			copy = maxfraglen - skb->len;
		if (copy <= 0) {
			char *data;
			unsigned int datalen;
			unsigned int fraglen;
			unsigned int fraggap;
			unsigned int alloclen;
			struct sk_buff *skb_prev;
alloc_new_skb:
			skb_prev = skb;
			if (skb_prev)
				fraggap = skb_prev->len - maxfraglen;
			else
				fraggap = 0;

			/*
			 * If remaining data exceeds the mtu,
			 * we know we need more fragment(s).
			 */
			datalen = length + fraggap;
			if (datalen > mtu - fragheaderlen)
				datalen = maxfraglen - fragheaderlen;
			fraglen = datalen + fragheaderlen;

970
			if ((flags & MSG_MORE) &&
971
			    !(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
972 973
				alloclen = mtu;
			else
974
				alloclen = fraglen;
L
Linus Torvalds 已提交
975

976 977
			alloclen += exthdrlen;

L
Linus Torvalds 已提交
978 979 980 981 982
			/* The last fragment gets additional space at tail.
			 * Note, with MSG_MORE we overallocate on fragments,
			 * because we have no idea what fragment will be
			 * the last.
			 */
983
			if (datalen == length + fraggap)
984
				alloclen += rt->dst.trailer_len;
985

L
Linus Torvalds 已提交
986
			if (transhdrlen) {
987
				skb = sock_alloc_send_skb(sk,
L
Linus Torvalds 已提交
988 989 990 991 992 993
						alloclen + hh_len + 15,
						(flags & MSG_DONTWAIT), &err);
			} else {
				skb = NULL;
				if (atomic_read(&sk->sk_wmem_alloc) <=
				    2 * sk->sk_sndbuf)
994
					skb = sock_wmalloc(sk,
L
Linus Torvalds 已提交
995 996
							   alloclen + hh_len + 15, 1,
							   sk->sk_allocation);
997
				if (unlikely(!skb))
L
Linus Torvalds 已提交
998 999
					err = -ENOBUFS;
			}
1000
			if (!skb)
L
Linus Torvalds 已提交
1001 1002 1003 1004 1005 1006 1007 1008
				goto error;

			/*
			 *	Fill in the control structures
			 */
			skb->ip_summed = csummode;
			skb->csum = 0;
			skb_reserve(skb, hh_len);
1009 1010

			/* only the initial fragment is time stamped */
1011
			skb_shinfo(skb)->tx_flags = cork->tx_flags;
1012
			cork->tx_flags = 0;
1013 1014
			skb_shinfo(skb)->tskey = tskey;
			tskey = 0;
L
Linus Torvalds 已提交
1015 1016 1017 1018

			/*
			 *	Find where to start putting bytes.
			 */
1019
			data = skb_put(skb, fraglen + exthdrlen);
1020
			skb_set_network_header(skb, exthdrlen);
1021 1022
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
1023
			data += fragheaderlen + exthdrlen;
L
Linus Torvalds 已提交
1024 1025 1026 1027 1028 1029 1030 1031

			if (fraggap) {
				skb->csum = skb_copy_and_csum_bits(
					skb_prev, maxfraglen,
					data + transhdrlen, fraggap, 0);
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
				data += fraggap;
1032
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
			}

			copy = datalen - transhdrlen - fraggap;
			if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
				err = -EFAULT;
				kfree_skb(skb);
				goto error;
			}

			offset += copy;
			length -= datalen - fraggap;
			transhdrlen = 0;
			exthdrlen = 0;
			csummode = CHECKSUM_NONE;

			/*
			 * Put the packet on the pending queue.
			 */
1051
			__skb_queue_tail(queue, skb);
L
Linus Torvalds 已提交
1052 1053 1054 1055 1056 1057
			continue;
		}

		if (copy > length)
			copy = length;

1058
		if (!(rt->dst.dev->features&NETIF_F_SG)) {
L
Linus Torvalds 已提交
1059 1060 1061
			unsigned int off;

			off = skb->len;
1062
			if (getfrag(from, skb_put(skb, copy),
L
Linus Torvalds 已提交
1063 1064 1065 1066 1067 1068 1069 1070
					offset, copy, off, skb) < 0) {
				__skb_trim(skb, off);
				err = -EFAULT;
				goto error;
			}
		} else {
			int i = skb_shinfo(skb)->nr_frags;

1071 1072
			err = -ENOMEM;
			if (!sk_page_frag_refill(sk, pfrag))
L
Linus Torvalds 已提交
1073
				goto error;
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084

			if (!skb_can_coalesce(skb, i, pfrag->page,
					      pfrag->offset)) {
				err = -EMSGSIZE;
				if (i == MAX_SKB_FRAGS)
					goto error;

				__skb_fill_page_desc(skb, i, pfrag->page,
						     pfrag->offset, 0);
				skb_shinfo(skb)->nr_frags = ++i;
				get_page(pfrag->page);
L
Linus Torvalds 已提交
1085
			}
1086 1087 1088 1089 1090 1091 1092 1093
			copy = min_t(int, copy, pfrag->size - pfrag->offset);
			if (getfrag(from,
				    page_address(pfrag->page) + pfrag->offset,
				    offset, copy, skb->len, skb) < 0)
				goto error_efault;

			pfrag->offset += copy;
			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
L
Linus Torvalds 已提交
1094 1095
			skb->len += copy;
			skb->data_len += copy;
1096 1097
			skb->truesize += copy;
			atomic_add(copy, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1098 1099 1100 1101 1102 1103 1104
		}
		offset += copy;
		length -= copy;
	}

	return 0;

1105 1106
error_efault:
	err = -EFAULT;
L
Linus Torvalds 已提交
1107
error:
1108
	cork->length -= length;
P
Pavel Emelyanov 已提交
1109
	IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
1110
	return err;
L
Linus Torvalds 已提交
1111 1112
}

1113 1114 1115
static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
			 struct ipcm_cookie *ipc, struct rtable **rtp)
{
1116
	struct ip_options_rcu *opt;
1117 1118 1119 1120 1121 1122 1123
	struct rtable *rt;

	/*
	 * setup for corking.
	 */
	opt = ipc->opt;
	if (opt) {
1124
		if (!cork->opt) {
1125 1126
			cork->opt = kmalloc(sizeof(struct ip_options) + 40,
					    sk->sk_allocation);
1127
			if (unlikely(!cork->opt))
1128 1129
				return -ENOBUFS;
		}
1130
		memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen);
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
		cork->flags |= IPCORK_OPT;
		cork->addr = ipc->addr;
	}
	rt = *rtp;
	if (unlikely(!rt))
		return -EFAULT;
	/*
	 * We steal reference to this route, caller should not release it
	 */
	*rtp = NULL;
1141 1142
	cork->fragsize = ip_sk_use_pmtu(sk) ?
			 dst_mtu(&rt->dst) : rt->dst.dev->mtu;
1143 1144
	cork->dst = &rt->dst;
	cork->length = 0;
1145 1146 1147
	cork->ttl = ipc->ttl;
	cork->tos = ipc->tos;
	cork->priority = ipc->priority;
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
	cork->tx_flags = ipc->tx_flags;

	return 0;
}

/*
 *	ip_append_data() and ip_append_page() can make one large IP datagram
 *	from many pieces of data. Each pieces will be holded on the socket
 *	until ip_push_pending_frames() is called. Each piece can be a page
 *	or non-page data.
 *
 *	Not only UDP, other transport protocols - e.g. raw sockets - can use
 *	this interface potentially.
 *
 *	LATER: length must be adjusted by pad at tail, when it is required.
 */
1164
int ip_append_data(struct sock *sk, struct flowi4 *fl4,
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
		   int getfrag(void *from, char *to, int offset, int len,
			       int odd, struct sk_buff *skb),
		   void *from, int length, int transhdrlen,
		   struct ipcm_cookie *ipc, struct rtable **rtp,
		   unsigned int flags)
{
	struct inet_sock *inet = inet_sk(sk);
	int err;

	if (flags&MSG_PROBE)
		return 0;

	if (skb_queue_empty(&sk->sk_write_queue)) {
1178
		err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp);
1179 1180 1181 1182 1183 1184
		if (err)
			return err;
	} else {
		transhdrlen = 0;
	}

1185 1186
	return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base,
				sk_page_frag(sk), getfrag,
1187 1188 1189
				from, length, transhdrlen, flags);
}

1190
ssize_t	ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,
L
Linus Torvalds 已提交
1191 1192 1193 1194 1195 1196
		       int offset, size_t size, int flags)
{
	struct inet_sock *inet = inet_sk(sk);
	struct sk_buff *skb;
	struct rtable *rt;
	struct ip_options *opt = NULL;
1197
	struct inet_cork *cork;
L
Linus Torvalds 已提交
1198 1199 1200 1201
	int hh_len;
	int mtu;
	int len;
	int err;
1202
	unsigned int maxfraglen, fragheaderlen, fraggap, maxnonfragsize;
L
Linus Torvalds 已提交
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212

	if (inet->hdrincl)
		return -EPERM;

	if (flags&MSG_PROBE)
		return 0;

	if (skb_queue_empty(&sk->sk_write_queue))
		return -EINVAL;

1213 1214 1215 1216
	cork = &inet->cork.base;
	rt = (struct rtable *)cork->dst;
	if (cork->flags & IPCORK_OPT)
		opt = cork->opt;
L
Linus Torvalds 已提交
1217

1218
	if (!(rt->dst.dev->features&NETIF_F_SG))
L
Linus Torvalds 已提交
1219 1220
		return -EOPNOTSUPP;

1221
	hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1222
	mtu = cork->fragsize;
L
Linus Torvalds 已提交
1223 1224 1225

	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
W
WANG Cong 已提交
1226
	maxnonfragsize = ip_sk_ignore_df(sk) ? 0xFFFF : mtu;
L
Linus Torvalds 已提交
1227

1228
	if (cork->length + size > maxnonfragsize - fragheaderlen) {
1229 1230
		ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
			       mtu - (opt ? opt->optlen : 0));
L
Linus Torvalds 已提交
1231 1232 1233
		return -EMSGSIZE;
	}

1234 1235
	skb = skb_peek_tail(&sk->sk_write_queue);
	if (!skb)
L
Linus Torvalds 已提交
1236 1237
		return -EINVAL;

1238
	cork->length += size;
1239 1240
	if ((size + skb->len > mtu) &&
	    (sk->sk_protocol == IPPROTO_UDP) &&
1241
	    (rt->dst.dev->features & NETIF_F_UFO)) {
1242
		skb_shinfo(skb)->gso_size = mtu - fragheaderlen;
H
Herbert Xu 已提交
1243
		skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1244
	}
1245

L
Linus Torvalds 已提交
1246
	while (size > 0) {
1247
		if (skb_is_gso(skb)) {
1248
			len = size;
1249
		} else {
1250 1251 1252 1253 1254 1255

			/* Check if the remaining data fits into current packet. */
			len = mtu - skb->len;
			if (len < size)
				len = maxfraglen - skb->len;
		}
L
Linus Torvalds 已提交
1256 1257 1258 1259 1260
		if (len <= 0) {
			struct sk_buff *skb_prev;
			int alloclen;

			skb_prev = skb;
1261
			fraggap = skb_prev->len - maxfraglen;
L
Linus Torvalds 已提交
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279

			alloclen = fragheaderlen + hh_len + fraggap + 15;
			skb = sock_wmalloc(sk, alloclen, 1, sk->sk_allocation);
			if (unlikely(!skb)) {
				err = -ENOBUFS;
				goto error;
			}

			/*
			 *	Fill in the control structures
			 */
			skb->ip_summed = CHECKSUM_NONE;
			skb->csum = 0;
			skb_reserve(skb, hh_len);

			/*
			 *	Find where to start putting bytes.
			 */
1280
			skb_put(skb, fragheaderlen + fraggap);
1281
			skb_reset_network_header(skb);
1282 1283
			skb->transport_header = (skb->network_header +
						 fragheaderlen);
L
Linus Torvalds 已提交
1284
			if (fraggap) {
1285 1286
				skb->csum = skb_copy_and_csum_bits(skb_prev,
								   maxfraglen,
1287
						    skb_transport_header(skb),
1288
								   fraggap, 0);
L
Linus Torvalds 已提交
1289 1290
				skb_prev->csum = csum_sub(skb_prev->csum,
							  skb->csum);
1291
				pskb_trim_unique(skb_prev, maxfraglen);
L
Linus Torvalds 已提交
1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
			}

			/*
			 * Put the packet on the pending queue.
			 */
			__skb_queue_tail(&sk->sk_write_queue, skb);
			continue;
		}

		if (len > size)
			len = size;
1303 1304

		if (skb_append_pagefrags(skb, page, offset, len)) {
L
Linus Torvalds 已提交
1305 1306 1307 1308 1309
			err = -EMSGSIZE;
			goto error;
		}

		if (skb->ip_summed == CHECKSUM_NONE) {
1310
			__wsum csum;
L
Linus Torvalds 已提交
1311 1312 1313 1314 1315 1316
			csum = csum_page(page, offset, len);
			skb->csum = csum_block_add(skb->csum, csum, skb->len);
		}

		skb->len += len;
		skb->data_len += len;
1317 1318
		skb->truesize += len;
		atomic_add(len, &sk->sk_wmem_alloc);
L
Linus Torvalds 已提交
1319 1320 1321 1322 1323 1324
		offset += len;
		size -= len;
	}
	return 0;

error:
1325
	cork->length -= size;
P
Pavel Emelyanov 已提交
1326
	IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1327 1328 1329
	return err;
}

1330
static void ip_cork_release(struct inet_cork *cork)
1331
{
1332 1333 1334 1335 1336
	cork->flags &= ~IPCORK_OPT;
	kfree(cork->opt);
	cork->opt = NULL;
	dst_release(cork->dst);
	cork->dst = NULL;
1337 1338
}

L
Linus Torvalds 已提交
1339 1340 1341 1342
/*
 *	Combined all pending IP fragments on the socket as one IP datagram
 *	and push them out.
 */
1343
struct sk_buff *__ip_make_skb(struct sock *sk,
1344
			      struct flowi4 *fl4,
1345 1346
			      struct sk_buff_head *queue,
			      struct inet_cork *cork)
L
Linus Torvalds 已提交
1347 1348 1349 1350
{
	struct sk_buff *skb, *tmp_skb;
	struct sk_buff **tail_skb;
	struct inet_sock *inet = inet_sk(sk);
1351
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
1352
	struct ip_options *opt = NULL;
1353
	struct rtable *rt = (struct rtable *)cork->dst;
L
Linus Torvalds 已提交
1354
	struct iphdr *iph;
1355
	__be16 df = 0;
L
Linus Torvalds 已提交
1356 1357
	__u8 ttl;

1358 1359
	skb = __skb_dequeue(queue);
	if (!skb)
L
Linus Torvalds 已提交
1360 1361 1362 1363
		goto out;
	tail_skb = &(skb_shinfo(skb)->frag_list);

	/* move skb->data to ip header from ext header */
1364
	if (skb->data < skb_network_header(skb))
1365
		__skb_pull(skb, skb_network_offset(skb));
1366
	while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1367
		__skb_pull(tmp_skb, skb_network_header_len(skb));
L
Linus Torvalds 已提交
1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
		*tail_skb = tmp_skb;
		tail_skb = &(tmp_skb->next);
		skb->len += tmp_skb->len;
		skb->data_len += tmp_skb->len;
		skb->truesize += tmp_skb->truesize;
		tmp_skb->destructor = NULL;
		tmp_skb->sk = NULL;
	}

	/* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
	 * to fragment the frame generated here. No matter, what transforms
	 * how transforms change size of the packet, it will come out.
	 */
W
WANG Cong 已提交
1381
	skb->ignore_df = ip_sk_ignore_df(sk);
L
Linus Torvalds 已提交
1382 1383

	/* DF bit is set when we want to see DF on outgoing frames.
W
WANG Cong 已提交
1384
	 * If ignore_df is set too, we still allow to fragment this frame
L
Linus Torvalds 已提交
1385
	 * locally. */
1386 1387
	if (inet->pmtudisc == IP_PMTUDISC_DO ||
	    inet->pmtudisc == IP_PMTUDISC_PROBE ||
1388 1389
	    (skb->len <= dst_mtu(&rt->dst) &&
	     ip_dont_fragment(sk, &rt->dst)))
L
Linus Torvalds 已提交
1390 1391
		df = htons(IP_DF);

1392 1393
	if (cork->flags & IPCORK_OPT)
		opt = cork->opt;
L
Linus Torvalds 已提交
1394

1395 1396 1397
	if (cork->ttl != 0)
		ttl = cork->ttl;
	else if (rt->rt_type == RTN_MULTICAST)
L
Linus Torvalds 已提交
1398 1399
		ttl = inet->mc_ttl;
	else
1400
		ttl = ip_select_ttl(inet, &rt->dst);
L
Linus Torvalds 已提交
1401

1402
	iph = ip_hdr(skb);
L
Linus Torvalds 已提交
1403 1404
	iph->version = 4;
	iph->ihl = 5;
1405
	iph->tos = (cork->tos != -1) ? cork->tos : inet->tos;
L
Linus Torvalds 已提交
1406 1407 1408
	iph->frag_off = df;
	iph->ttl = ttl;
	iph->protocol = sk->sk_protocol;
1409
	ip_copy_addrs(iph, fl4);
1410
	ip_select_ident(net, skb, sk);
L
Linus Torvalds 已提交
1411

1412 1413 1414 1415 1416
	if (opt) {
		iph->ihl += opt->optlen>>2;
		ip_options_build(skb, opt, cork->addr, rt, 0);
	}

1417
	skb->priority = (cork->tos != -1) ? cork->priority: sk->sk_priority;
1418
	skb->mark = sk->sk_mark;
1419 1420 1421 1422
	/*
	 * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
	 * on dst refcount
	 */
1423
	cork->dst = NULL;
1424
	skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
1425

1426
	if (iph->protocol == IPPROTO_ICMP)
1427
		icmp_out_count(net, ((struct icmphdr *)
1428 1429
			skb_transport_header(skb))->type);

1430 1431 1432 1433 1434
	ip_cork_release(cork);
out:
	return skb;
}

E
Eric Dumazet 已提交
1435
int ip_send_skb(struct net *net, struct sk_buff *skb)
1436 1437 1438
{
	int err;

H
Herbert Xu 已提交
1439
	err = ip_local_out(skb);
L
Linus Torvalds 已提交
1440 1441
	if (err) {
		if (err > 0)
E
Eric Dumazet 已提交
1442
			err = net_xmit_errno(err);
L
Linus Torvalds 已提交
1443
		if (err)
1444
			IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
L
Linus Torvalds 已提交
1445 1446 1447 1448 1449
	}

	return err;
}

1450
int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4)
1451
{
1452 1453
	struct sk_buff *skb;

1454
	skb = ip_finish_skb(sk, fl4);
1455 1456 1457 1458
	if (!skb)
		return 0;

	/* Netfilter gets whole the not fragmented skb. */
E
Eric Dumazet 已提交
1459
	return ip_send_skb(sock_net(sk), skb);
1460 1461
}

L
Linus Torvalds 已提交
1462 1463 1464
/*
 *	Throw away all pending data on the socket.
 */
1465 1466 1467
static void __ip_flush_pending_frames(struct sock *sk,
				      struct sk_buff_head *queue,
				      struct inet_cork *cork)
L
Linus Torvalds 已提交
1468 1469 1470
{
	struct sk_buff *skb;

1471
	while ((skb = __skb_dequeue_tail(queue)) != NULL)
L
Linus Torvalds 已提交
1472 1473
		kfree_skb(skb);

1474 1475 1476 1477 1478
	ip_cork_release(cork);
}

void ip_flush_pending_frames(struct sock *sk)
{
1479
	__ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
L
Linus Torvalds 已提交
1480 1481
}

1482
struct sk_buff *ip_make_skb(struct sock *sk,
1483
			    struct flowi4 *fl4,
1484 1485 1486 1487 1488 1489
			    int getfrag(void *from, char *to, int offset,
					int len, int odd, struct sk_buff *skb),
			    void *from, int length, int transhdrlen,
			    struct ipcm_cookie *ipc, struct rtable **rtp,
			    unsigned int flags)
{
1490
	struct inet_cork cork;
1491 1492 1493 1494 1495 1496 1497 1498
	struct sk_buff_head queue;
	int err;

	if (flags & MSG_PROBE)
		return NULL;

	__skb_queue_head_init(&queue);

1499 1500
	cork.flags = 0;
	cork.addr = 0;
1501
	cork.opt = NULL;
1502 1503 1504 1505
	err = ip_setup_cork(sk, &cork, ipc, rtp);
	if (err)
		return ERR_PTR(err);

1506 1507
	err = __ip_append_data(sk, fl4, &queue, &cork,
			       &current->task_frag, getfrag,
1508 1509 1510 1511 1512 1513
			       from, length, transhdrlen, flags);
	if (err) {
		__ip_flush_pending_frames(sk, &queue, &cork);
		return ERR_PTR(err);
	}

1514
	return __ip_make_skb(sk, fl4, &queue, &cork);
1515
}
L
Linus Torvalds 已提交
1516 1517 1518 1519

/*
 *	Fetch data from kernel space and fill in checksum if needed.
 */
1520
static int ip_reply_glue_bits(void *dptr, char *to, int offset,
L
Linus Torvalds 已提交
1521 1522
			      int len, int odd, struct sk_buff *skb)
{
1523
	__wsum csum;
L
Linus Torvalds 已提交
1524 1525 1526

	csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
	skb->csum = csum_block_add(skb->csum, csum, odd);
1527
	return 0;
L
Linus Torvalds 已提交
1528 1529
}

1530
/*
L
Linus Torvalds 已提交
1531
 *	Generic function to send a packet as reply to another packet.
1532
 *	Used to send some TCP resets/acks so far.
L
Linus Torvalds 已提交
1533
 */
1534
void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb,
1535 1536 1537
			   const struct ip_options *sopt,
			   __be32 daddr, __be32 saddr,
			   const struct ip_reply_arg *arg,
1538
			   unsigned int len)
L
Linus Torvalds 已提交
1539
{
1540
	struct ip_options_data replyopts;
L
Linus Torvalds 已提交
1541
	struct ipcm_cookie ipc;
1542
	struct flowi4 fl4;
E
Eric Dumazet 已提交
1543
	struct rtable *rt = skb_rtable(skb);
1544
	struct net *net = sock_net(sk);
1545
	struct sk_buff *nskb;
1546
	int err;
1547
	int oif;
L
Linus Torvalds 已提交
1548

1549
	if (__ip_options_echo(&replyopts.opt.opt, skb, sopt))
L
Linus Torvalds 已提交
1550 1551
		return;

1552
	ipc.addr = daddr;
L
Linus Torvalds 已提交
1553
	ipc.opt = NULL;
1554
	ipc.tx_flags = 0;
1555 1556
	ipc.ttl = 0;
	ipc.tos = -1;
L
Linus Torvalds 已提交
1557

1558
	if (replyopts.opt.opt.optlen) {
L
Linus Torvalds 已提交
1559 1560
		ipc.opt = &replyopts.opt;

1561 1562
		if (replyopts.opt.opt.srr)
			daddr = replyopts.opt.opt.faddr;
L
Linus Torvalds 已提交
1563 1564
	}

1565
	oif = arg->bound_dev_if;
1566
	if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
1567 1568 1569
		oif = skb->skb_iif;

	flowi4_init_output(&fl4, oif,
1570
			   IP4_REPLY_MARK(net, skb->mark),
1571
			   RT_TOS(arg->tos),
1572
			   RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol,
1573
			   ip_reply_arg_flowi_flags(arg),
1574
			   daddr, saddr,
1575 1576
			   tcp_hdr(skb)->source, tcp_hdr(skb)->dest);
	security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
1577
	rt = ip_route_output_key(net, &fl4);
1578 1579
	if (IS_ERR(rt))
		return;
L
Linus Torvalds 已提交
1580

1581
	inet_sk(sk)->tos = arg->tos;
L
Linus Torvalds 已提交
1582 1583

	sk->sk_priority = skb->priority;
1584
	sk->sk_protocol = ip_hdr(skb)->protocol;
1585
	sk->sk_bound_dev_if = arg->bound_dev_if;
1586
	sk->sk_sndbuf = sysctl_wmem_default;
1587 1588 1589 1590 1591 1592 1593
	err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base,
			     len, 0, &ipc, &rt, MSG_DONTWAIT);
	if (unlikely(err)) {
		ip_flush_pending_frames(sk);
		goto out;
	}

1594 1595
	nskb = skb_peek(&sk->sk_write_queue);
	if (nskb) {
L
Linus Torvalds 已提交
1596
		if (arg->csumoffset >= 0)
1597 1598
			*((__sum16 *)skb_transport_header(nskb) +
			  arg->csumoffset) = csum_fold(csum_add(nskb->csum,
1599
								arg->csum));
1600 1601
		nskb->ip_summed = CHECKSUM_NONE;
		skb_set_queue_mapping(nskb, skb_get_queue_mapping(skb));
1602
		ip_push_pending_frames(sk, &fl4);
L
Linus Torvalds 已提交
1603
	}
1604
out:
L
Linus Torvalds 已提交
1605 1606 1607 1608 1609 1610 1611 1612
	ip_rt_put(rt);
}

void __init ip_init(void)
{
	ip_rt_init();
	inet_initpeers();

1613 1614
#if defined(CONFIG_IP_MULTICAST)
	igmp_mc_init();
L
Linus Torvalds 已提交
1615 1616
#endif
}