route.c 71.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		ROUTE - implementation of the IP router.
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Linus Torvalds, <Linus.Torvalds@helsinki.fi>
 *		Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 * Fixes:
 *		Alan Cox	:	Verify area fixes.
 *		Alan Cox	:	cli() protects routing changes
 *		Rui Oliveira	:	ICMP routing table updates
 *		(rco@di.uminho.pt)	Routing table insertion and update
 *		Linus Torvalds	:	Rewrote bits to be sensible
 *		Alan Cox	:	Added BSD route gw semantics
21
 *		Alan Cox	:	Super /proc >4K
L
Linus Torvalds 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
 *		Alan Cox	:	MTU in route table
 *		Alan Cox	: 	MSS actually. Also added the window
 *					clamper.
 *		Sam Lantinga	:	Fixed route matching in rt_del()
 *		Alan Cox	:	Routing cache support.
 *		Alan Cox	:	Removed compatibility cruft.
 *		Alan Cox	:	RTF_REJECT support.
 *		Alan Cox	:	TCP irtt support.
 *		Jonathan Naylor	:	Added Metric support.
 *	Miquel van Smoorenburg	:	BSD API fixes.
 *	Miquel van Smoorenburg	:	Metrics.
 *		Alan Cox	:	Use __u32 properly
 *		Alan Cox	:	Aligned routing errors more closely with BSD
 *					our system is still very different.
 *		Alan Cox	:	Faster /proc handling
 *	Alexey Kuznetsov	:	Massive rework to support tree based routing,
 *					routing caches and better behaviour.
39
 *
L
Linus Torvalds 已提交
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
 *		Olaf Erb	:	irtt wasn't being copied right.
 *		Bjorn Ekwall	:	Kerneld route support.
 *		Alan Cox	:	Multicast fixed (I hope)
 * 		Pavel Krauz	:	Limited broadcast fixed
 *		Mike McLagan	:	Routing by source
 *	Alexey Kuznetsov	:	End of old history. Split to fib.c and
 *					route.c and rewritten from scratch.
 *		Andi Kleen	:	Load-limit warning messages.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
 *	Vitaly E. Lavrov	:	Race condition in ip_route_input_slow.
 *	Tobias Ringstrom	:	Uninitialized res.type in ip_route_output_slow.
 *	Vladimir V. Ivanov	:	IP rule info (flowid) is really useful.
 *		Marc Boucher	:	routing by fwmark
 *	Robert Olsson		:	Added rt_cache statistics
 *	Arnaldo C. Melo		:	Convert proc stuff to seq_file
55
 *	Eric Dumazet		:	hashed spinlocks and rt_check_expire() fixes.
56 57
 * 	Ilia Sotnikov		:	Ignore TOS on PMTUD and Redirect
 * 	Ilia Sotnikov		:	Removed TOS from hash calculations
L
Linus Torvalds 已提交
58 59 60 61 62 63 64
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 */

65 66
#define pr_fmt(fmt) "IPv4: " fmt

L
Linus Torvalds 已提交
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
#include <linux/module.h>
#include <asm/uaccess.h>
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include <linux/pkt_sched.h>
#include <linux/mroute.h>
#include <linux/netfilter_ipv4.h>
#include <linux/random.h>
#include <linux/rcupdate.h>
#include <linux/times.h>
91
#include <linux/slab.h>
E
Eric Dumazet 已提交
92
#include <linux/jhash.h>
93
#include <net/dst.h>
94
#include <net/dst_metadata.h>
95
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104 105
#include <net/protocol.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/inetpeer.h>
#include <net/sock.h>
#include <net/ip_fib.h>
#include <net/arp.h>
#include <net/tcp.h>
#include <net/icmp.h>
#include <net/xfrm.h>
106
#include <net/lwtunnel.h>
107
#include <net/netevent.h>
108
#include <net/rtnetlink.h>
L
Linus Torvalds 已提交
109 110
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
111
#include <linux/kmemleak.h>
L
Linus Torvalds 已提交
112
#endif
113
#include <net/secure_seq.h>
114
#include <net/ip_tunnels.h>
115
#include <net/l3mdev.h>
L
Linus Torvalds 已提交
116

117
#define RT_FL_TOS(oldflp4) \
118
	((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
L
Linus Torvalds 已提交
119 120 121 122

#define RT_GC_TIMEOUT (300*HZ)

static int ip_rt_max_size;
123 124 125 126 127 128 129 130
static int ip_rt_redirect_number __read_mostly	= 9;
static int ip_rt_redirect_load __read_mostly	= HZ / 50;
static int ip_rt_redirect_silence __read_mostly	= ((HZ / 50) << (9 + 1));
static int ip_rt_error_cost __read_mostly	= HZ;
static int ip_rt_error_burst __read_mostly	= 5 * HZ;
static int ip_rt_mtu_expires __read_mostly	= 10 * 60 * HZ;
static int ip_rt_min_pmtu __read_mostly		= 512 + 20 + 20;
static int ip_rt_min_advmss __read_mostly	= 256;
131

132
static int ip_rt_gc_timeout __read_mostly	= RT_GC_TIMEOUT;
L
Linus Torvalds 已提交
133 134 135 136 137
/*
 *	Interface to generic destination cache.
 */

static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
138
static unsigned int	 ipv4_default_advmss(const struct dst_entry *dst);
139
static unsigned int	 ipv4_mtu(const struct dst_entry *dst);
L
Linus Torvalds 已提交
140 141
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
static void		 ipv4_link_failure(struct sk_buff *skb);
142 143 144 145
static void		 ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
					   struct sk_buff *skb, u32 mtu);
static void		 ip_do_redirect(struct dst_entry *dst, struct sock *sk,
					struct sk_buff *skb);
146
static void		ipv4_dst_destroy(struct dst_entry *dst);
L
Linus Torvalds 已提交
147

148 149
static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
{
150 151
	WARN_ON(1);
	return NULL;
152 153
}

154 155 156
static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
					   struct sk_buff *skb,
					   const void *daddr);
157

L
Linus Torvalds 已提交
158 159 160
static struct dst_ops ipv4_dst_ops = {
	.family =		AF_INET,
	.check =		ipv4_dst_check,
161
	.default_advmss =	ipv4_default_advmss,
162
	.mtu =			ipv4_mtu,
163
	.cow_metrics =		ipv4_cow_metrics,
164
	.destroy =		ipv4_dst_destroy,
L
Linus Torvalds 已提交
165 166 167
	.negative_advice =	ipv4_negative_advice,
	.link_failure =		ipv4_link_failure,
	.update_pmtu =		ip_rt_update_pmtu,
168
	.redirect =		ip_do_redirect,
169
	.local_out =		__ip_local_out,
170
	.neigh_lookup =		ipv4_neigh_lookup,
L
Linus Torvalds 已提交
171 172 173 174
};

#define ECN_OR_COST(class)	TC_PRIO_##class

175
const __u8 ip_tos2prio[16] = {
L
Linus Torvalds 已提交
176
	TC_PRIO_BESTEFFORT,
D
Dan Siemon 已提交
177
	ECN_OR_COST(BESTEFFORT),
L
Linus Torvalds 已提交
178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
	TC_PRIO_BESTEFFORT,
	ECN_OR_COST(BESTEFFORT),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK)
};
193
EXPORT_SYMBOL(ip_tos2prio);
L
Linus Torvalds 已提交
194

195
static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
196
#define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
L
Linus Torvalds 已提交
197 198 199 200

#ifdef CONFIG_PROC_FS
static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
{
201
	if (*pos)
D
David S. Miller 已提交
202
		return NULL;
203
	return SEQ_START_TOKEN;
L
Linus Torvalds 已提交
204 205 206 207 208
}

static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	++*pos;
D
David S. Miller 已提交
209
	return NULL;
L
Linus Torvalds 已提交
210 211 212 213 214 215 216 217 218 219 220 221 222
}

static void rt_cache_seq_stop(struct seq_file *seq, void *v)
{
}

static int rt_cache_seq_show(struct seq_file *seq, void *v)
{
	if (v == SEQ_START_TOKEN)
		seq_printf(seq, "%-127s\n",
			   "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
			   "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
			   "HHUptod\tSpecDst");
223
	return 0;
L
Linus Torvalds 已提交
224 225
}

226
static const struct seq_operations rt_cache_seq_ops = {
L
Linus Torvalds 已提交
227 228 229 230 231 232 233 234
	.start  = rt_cache_seq_start,
	.next   = rt_cache_seq_next,
	.stop   = rt_cache_seq_stop,
	.show   = rt_cache_seq_show,
};

static int rt_cache_seq_open(struct inode *inode, struct file *file)
{
D
David S. Miller 已提交
235
	return seq_open(file, &rt_cache_seq_ops);
L
Linus Torvalds 已提交
236 237
}

238
static const struct file_operations rt_cache_seq_fops = {
L
Linus Torvalds 已提交
239 240 241 242
	.owner	 = THIS_MODULE,
	.open	 = rt_cache_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
D
David S. Miller 已提交
243
	.release = seq_release,
L
Linus Torvalds 已提交
244 245 246 247 248 249 250 251 252 253
};


static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
{
	int cpu;

	if (*pos == 0)
		return SEQ_START_TOKEN;

254
	for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
255 256 257
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
258
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
259 260 261 262 263 264 265 266
	}
	return NULL;
}

static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	int cpu;

267
	for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
268 269 270
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
271
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
272 273
	}
	return NULL;
274

L
Linus Torvalds 已提交
275 276 277 278 279 280 281 282 283 284 285 286
}

static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
{

}

static int rt_cpu_seq_show(struct seq_file *seq, void *v)
{
	struct rt_cache_stat *st = v;

	if (v == SEQ_START_TOKEN) {
287
		seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
L
Linus Torvalds 已提交
288 289
		return 0;
	}
290

L
Linus Torvalds 已提交
291 292
	seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
		   " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
293
		   dst_entries_get_slow(&ipv4_dst_ops),
E
Eric Dumazet 已提交
294
		   0, /* st->in_hit */
L
Linus Torvalds 已提交
295 296 297 298 299 300 301
		   st->in_slow_tot,
		   st->in_slow_mc,
		   st->in_no_route,
		   st->in_brd,
		   st->in_martian_dst,
		   st->in_martian_src,

E
Eric Dumazet 已提交
302
		   0, /* st->out_hit */
L
Linus Torvalds 已提交
303
		   st->out_slow_tot,
304
		   st->out_slow_mc,
L
Linus Torvalds 已提交
305

E
Eric Dumazet 已提交
306 307 308 309 310 311
		   0, /* st->gc_total */
		   0, /* st->gc_ignored */
		   0, /* st->gc_goal_miss */
		   0, /* st->gc_dst_overflow */
		   0, /* st->in_hlist_search */
		   0  /* st->out_hlist_search */
L
Linus Torvalds 已提交
312 313 314 315
		);
	return 0;
}

316
static const struct seq_operations rt_cpu_seq_ops = {
L
Linus Torvalds 已提交
317 318 319 320 321 322 323 324 325 326 327 328
	.start  = rt_cpu_seq_start,
	.next   = rt_cpu_seq_next,
	.stop   = rt_cpu_seq_stop,
	.show   = rt_cpu_seq_show,
};


static int rt_cpu_seq_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &rt_cpu_seq_ops);
}

329
static const struct file_operations rt_cpu_seq_fops = {
L
Linus Torvalds 已提交
330 331 332 333 334 335 336
	.owner	 = THIS_MODULE,
	.open	 = rt_cpu_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
	.release = seq_release,
};

337
#ifdef CONFIG_IP_ROUTE_CLASSID
338
static int rt_acct_proc_show(struct seq_file *m, void *v)
339
{
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
	struct ip_rt_acct *dst, *src;
	unsigned int i, j;

	dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
	if (!dst)
		return -ENOMEM;

	for_each_possible_cpu(i) {
		src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
		for (j = 0; j < 256; j++) {
			dst[j].o_bytes   += src[j].o_bytes;
			dst[j].o_packets += src[j].o_packets;
			dst[j].i_bytes   += src[j].i_bytes;
			dst[j].i_packets += src[j].i_packets;
		}
355 356
	}

357 358 359 360
	seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
	kfree(dst);
	return 0;
}
361

362 363 364
static int rt_acct_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, rt_acct_proc_show, NULL);
365
}
366 367 368 369 370 371 372 373

static const struct file_operations rt_acct_proc_fops = {
	.owner		= THIS_MODULE,
	.open		= rt_acct_proc_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};
374
#endif
375

376
static int __net_init ip_rt_do_proc_init(struct net *net)
377 378 379
{
	struct proc_dir_entry *pde;

380 381
	pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
			  &rt_cache_seq_fops);
382 383 384
	if (!pde)
		goto err1;

385 386
	pde = proc_create("rt_cache", S_IRUGO,
			  net->proc_net_stat, &rt_cpu_seq_fops);
387 388 389
	if (!pde)
		goto err2;

390
#ifdef CONFIG_IP_ROUTE_CLASSID
391
	pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
392 393 394 395 396
	if (!pde)
		goto err3;
#endif
	return 0;

397
#ifdef CONFIG_IP_ROUTE_CLASSID
398 399 400 401 402 403 404 405
err3:
	remove_proc_entry("rt_cache", net->proc_net_stat);
#endif
err2:
	remove_proc_entry("rt_cache", net->proc_net);
err1:
	return -ENOMEM;
}
406 407 408 409 410

static void __net_exit ip_rt_do_proc_exit(struct net *net)
{
	remove_proc_entry("rt_cache", net->proc_net_stat);
	remove_proc_entry("rt_cache", net->proc_net);
411
#ifdef CONFIG_IP_ROUTE_CLASSID
412
	remove_proc_entry("rt_acct", net->proc_net);
413
#endif
414 415 416 417 418 419 420 421 422 423 424 425
}

static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
	.init = ip_rt_do_proc_init,
	.exit = ip_rt_do_proc_exit,
};

static int __init ip_rt_proc_init(void)
{
	return register_pernet_subsys(&ip_rt_proc_ops);
}

426
#else
427
static inline int ip_rt_proc_init(void)
428 429 430
{
	return 0;
}
L
Linus Torvalds 已提交
431
#endif /* CONFIG_PROC_FS */
432

433
static inline bool rt_is_expired(const struct rtable *rth)
434
{
F
fan.du 已提交
435
	return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
436 437
}

438
void rt_cache_flush(struct net *net)
L
Linus Torvalds 已提交
439
{
F
fan.du 已提交
440
	rt_genid_bump_ipv4(net);
E
Eric Dumazet 已提交
441 442
}

443 444 445
static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
					   struct sk_buff *skb,
					   const void *daddr)
D
David Miller 已提交
446
{
447 448
	struct net_device *dev = dst->dev;
	const __be32 *pkey = daddr;
449
	const struct rtable *rt;
D
David Miller 已提交
450 451
	struct neighbour *n;

452
	rt = (const struct rtable *) dst;
453
	if (rt->rt_gateway)
454
		pkey = (const __be32 *) &rt->rt_gateway;
455 456
	else if (skb)
		pkey = &ip_hdr(skb)->daddr;
457

458
	n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
459 460
	if (n)
		return n;
461
	return neigh_create(&arp_tbl, pkey, dev);
462 463
}

464 465
#define IP_IDENTS_SZ 2048u

E
Eric Dumazet 已提交
466 467
static atomic_t *ip_idents __read_mostly;
static u32 *ip_tstamps __read_mostly;
468 469 470 471 472 473 474

/* In order to protect privacy, we add a perturbation to identifiers
 * if one generator is seldom used. This makes hard for an attacker
 * to infer how many packets were sent between two points in time.
 */
u32 ip_idents_reserve(u32 hash, int segs)
{
E
Eric Dumazet 已提交
475 476 477
	u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
	atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
	u32 old = ACCESS_ONCE(*p_tstamp);
478
	u32 now = (u32)jiffies;
479
	u32 new, delta = 0;
480

E
Eric Dumazet 已提交
481
	if (old != now && cmpxchg(p_tstamp, old, now) == old)
482 483
		delta = prandom_u32_max(now - old);

484 485 486 487 488 489 490
	/* Do not use atomic_add_return() as it makes UBSAN unhappy */
	do {
		old = (u32)atomic_read(p_id);
		new = old + delta + segs;
	} while (atomic_cmpxchg(p_id, old, new) != old);

	return new - segs;
491 492
}
EXPORT_SYMBOL(ip_idents_reserve);
L
Linus Torvalds 已提交
493

494
void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
L
Linus Torvalds 已提交
495
{
E
Eric Dumazet 已提交
496 497
	static u32 ip_idents_hashrnd __read_mostly;
	u32 hash, id;
L
Linus Torvalds 已提交
498

E
Eric Dumazet 已提交
499
	net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
L
Linus Torvalds 已提交
500

501 502
	hash = jhash_3words((__force u32)iph->daddr,
			    (__force u32)iph->saddr,
503
			    iph->protocol ^ net_hash_mix(net),
504
			    ip_idents_hashrnd);
E
Eric Dumazet 已提交
505 506
	id = ip_idents_reserve(hash, segs);
	iph->id = htons(id);
L
Linus Torvalds 已提交
507
}
E
Eric Dumazet 已提交
508
EXPORT_SYMBOL(__ip_select_ident);
L
Linus Torvalds 已提交
509

E
Eric Dumazet 已提交
510
static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
			     const struct iphdr *iph,
			     int oif, u8 tos,
			     u8 prot, u32 mark, int flow_flags)
{
	if (sk) {
		const struct inet_sock *inet = inet_sk(sk);

		oif = sk->sk_bound_dev_if;
		mark = sk->sk_mark;
		tos = RT_CONN_FLAGS(sk);
		prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
	}
	flowi4_init_output(fl4, oif, mark, tos,
			   RT_SCOPE_UNIVERSE, prot,
			   flow_flags,
			   iph->daddr, iph->saddr, 0, 0);
}

E
Eric Dumazet 已提交
529 530
static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
			       const struct sock *sk)
531 532 533 534 535 536 537 538 539 540
{
	const struct iphdr *iph = ip_hdr(skb);
	int oif = skb->dev->ifindex;
	u8 tos = RT_TOS(iph->tos);
	u8 prot = iph->protocol;
	u32 mark = skb->mark;

	__build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
}

E
Eric Dumazet 已提交
541
static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
542 543
{
	const struct inet_sock *inet = inet_sk(sk);
E
Eric Dumazet 已提交
544
	const struct ip_options_rcu *inet_opt;
545 546 547 548 549 550 551 552 553 554 555 556 557 558
	__be32 daddr = inet->inet_daddr;

	rcu_read_lock();
	inet_opt = rcu_dereference(inet->inet_opt);
	if (inet_opt && inet_opt->opt.srr)
		daddr = inet_opt->opt.faddr;
	flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
			   RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
			   inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
			   inet_sk_flowi_flags(sk),
			   daddr, inet->inet_saddr, 0, 0);
	rcu_read_unlock();
}

E
Eric Dumazet 已提交
559 560
static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
				 const struct sk_buff *skb)
561 562 563 564 565 566 567
{
	if (skb)
		build_skb_flow_key(fl4, skb, sk);
	else
		build_sk_flow_key(fl4, sk);
}

568 569 570 571 572 573
static inline void rt_free(struct rtable *rt)
{
	call_rcu(&rt->dst.rcu_head, dst_rcu_free);
}

static DEFINE_SPINLOCK(fnhe_lock);
574

575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
{
	struct rtable *rt;

	rt = rcu_dereference(fnhe->fnhe_rth_input);
	if (rt) {
		RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
		rt_free(rt);
	}
	rt = rcu_dereference(fnhe->fnhe_rth_output);
	if (rt) {
		RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
		rt_free(rt);
	}
}

591
static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
592 593 594 595 596 597 598 599 600
{
	struct fib_nh_exception *fnhe, *oldest;

	oldest = rcu_dereference(hash->chain);
	for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
		if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
			oldest = fnhe;
	}
601
	fnhe_flush_routes(oldest);
602 603 604
	return oldest;
}

605 606
static inline u32 fnhe_hashfun(__be32 daddr)
{
E
Eric Dumazet 已提交
607
	static u32 fnhe_hashrnd __read_mostly;
608 609
	u32 hval;

E
Eric Dumazet 已提交
610 611 612
	net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
	hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
	return hash_32(hval, FNHE_HASH_SHIFT);
613 614
}

615 616 617 618 619 620 621 622 623 624 625 626
static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
{
	rt->rt_pmtu = fnhe->fnhe_pmtu;
	rt->dst.expires = fnhe->fnhe_expires;

	if (fnhe->fnhe_gw) {
		rt->rt_flags |= RTCF_REDIRECTED;
		rt->rt_gateway = fnhe->fnhe_gw;
		rt->rt_uses_gateway = 1;
	}
}

627 628
static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
				  u32 pmtu, unsigned long expires)
629
{
630
	struct fnhe_hash_bucket *hash;
631
	struct fib_nh_exception *fnhe;
632 633
	struct rtable *rt;
	unsigned int i;
634
	int depth;
635 636
	u32 hval = fnhe_hashfun(daddr);

637
	spin_lock_bh(&fnhe_lock);
638

639
	hash = rcu_dereference(nh->nh_exceptions);
640
	if (!hash) {
641
		hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
642
		if (!hash)
643
			goto out_unlock;
644
		rcu_assign_pointer(nh->nh_exceptions, hash);
645 646 647 648 649 650 651 652
	}

	hash += hval;

	depth = 0;
	for (fnhe = rcu_dereference(hash->chain); fnhe;
	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
		if (fnhe->fnhe_daddr == daddr)
653
			break;
654 655 656
		depth++;
	}

657 658 659 660 661
	if (fnhe) {
		if (gw)
			fnhe->fnhe_gw = gw;
		if (pmtu) {
			fnhe->fnhe_pmtu = pmtu;
662
			fnhe->fnhe_expires = max(1UL, expires);
663
		}
664
		/* Update all cached dsts too */
665 666 667 668
		rt = rcu_dereference(fnhe->fnhe_rth_input);
		if (rt)
			fill_route_from_fnhe(rt, fnhe);
		rt = rcu_dereference(fnhe->fnhe_rth_output);
669 670
		if (rt)
			fill_route_from_fnhe(rt, fnhe);
671 672 673 674 675 676 677 678 679 680 681
	} else {
		if (depth > FNHE_RECLAIM_DEPTH)
			fnhe = fnhe_oldest(hash);
		else {
			fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
			if (!fnhe)
				goto out_unlock;

			fnhe->fnhe_next = hash->chain;
			rcu_assign_pointer(hash->chain, fnhe);
		}
682
		fnhe->fnhe_genid = fnhe_genid(dev_net(nh->nh_dev));
683 684 685 686
		fnhe->fnhe_daddr = daddr;
		fnhe->fnhe_gw = gw;
		fnhe->fnhe_pmtu = pmtu;
		fnhe->fnhe_expires = expires;
687 688 689 690 691

		/* Exception created; mark the cached routes for the nexthop
		 * stale, so anyone caching it rechecks if this exception
		 * applies to them.
		 */
692 693 694 695
		rt = rcu_dereference(nh->nh_rth_input);
		if (rt)
			rt->dst.obsolete = DST_OBSOLETE_KILL;

696 697 698 699 700 701 702
		for_each_possible_cpu(i) {
			struct rtable __rcu **prt;
			prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
			rt = rcu_dereference(*prt);
			if (rt)
				rt->dst.obsolete = DST_OBSOLETE_KILL;
		}
703 704 705
	}

	fnhe->fnhe_stamp = jiffies;
706 707

out_unlock:
708
	spin_unlock_bh(&fnhe_lock);
709 710
}

711 712
static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
			     bool kill_route)
L
Linus Torvalds 已提交
713
{
714
	__be32 new_gw = icmp_hdr(skb)->un.gateway;
715
	__be32 old_gw = ip_hdr(skb)->saddr;
716 717
	struct net_device *dev = skb->dev;
	struct in_device *in_dev;
718
	struct fib_result res;
719
	struct neighbour *n;
720
	struct net *net;
L
Linus Torvalds 已提交
721

722 723 724 725 726 727 728 729 730 731 732
	switch (icmp_hdr(skb)->code & 7) {
	case ICMP_REDIR_NET:
	case ICMP_REDIR_NETTOS:
	case ICMP_REDIR_HOST:
	case ICMP_REDIR_HOSTTOS:
		break;

	default:
		return;
	}

733 734 735 736 737 738 739
	if (rt->rt_gateway != old_gw)
		return;

	in_dev = __in_dev_get_rcu(dev);
	if (!in_dev)
		return;

740
	net = dev_net(dev);
741 742 743
	if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
	    ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
	    ipv4_is_zeronet(new_gw))
L
Linus Torvalds 已提交
744 745 746 747 748 749 750 751
		goto reject_redirect;

	if (!IN_DEV_SHARED_MEDIA(in_dev)) {
		if (!inet_addr_onlink(in_dev, new_gw, old_gw))
			goto reject_redirect;
		if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
			goto reject_redirect;
	} else {
752
		if (inet_addr_type(net, new_gw) != RTN_UNICAST)
L
Linus Torvalds 已提交
753 754 755
			goto reject_redirect;
	}

756
	n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
757
	if (!IS_ERR(n)) {
758 759 760
		if (!(n->nud_state & NUD_VALID)) {
			neigh_event_send(n, NULL);
		} else {
761
			if (fib_lookup(net, fl4, &res, 0) == 0) {
762 763
				struct fib_nh *nh = &FIB_RES_NH(res);

764
				update_or_create_fnhe(nh, fl4->daddr, new_gw,
765
						0, jiffies + ip_rt_gc_timeout);
766
			}
767 768
			if (kill_route)
				rt->dst.obsolete = DST_OBSOLETE_KILL;
769 770 771 772 773 774 775 776
			call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
		}
		neigh_release(n);
	}
	return;

reject_redirect:
#ifdef CONFIG_IP_ROUTE_VERBOSE
777 778 779 780 781
	if (IN_DEV_LOG_MARTIANS(in_dev)) {
		const struct iphdr *iph = (const struct iphdr *) skb->data;
		__be32 daddr = iph->daddr;
		__be32 saddr = iph->saddr;

782 783 784 785
		net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
				     "  Advised path = %pI4 -> %pI4\n",
				     &old_gw, dev->name, &new_gw,
				     &saddr, &daddr);
786
	}
787 788 789 790
#endif
	;
}

791 792 793 794
static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
{
	struct rtable *rt;
	struct flowi4 fl4;
795 796 797 798 799
	const struct iphdr *iph = (const struct iphdr *) skb->data;
	int oif = skb->dev->ifindex;
	u8 tos = RT_TOS(iph->tos);
	u8 prot = iph->protocol;
	u32 mark = skb->mark;
800 801 802

	rt = (struct rtable *) dst;

803
	__build_flow_key(&fl4, sk, iph, oif, tos, prot, mark, 0);
804
	__ip_do_redirect(rt, skb, &fl4, true);
805 806
}

L
Linus Torvalds 已提交
807 808
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
{
809
	struct rtable *rt = (struct rtable *)dst;
L
Linus Torvalds 已提交
810 811 812
	struct dst_entry *ret = dst;

	if (rt) {
T
Timo Teräs 已提交
813
		if (dst->obsolete > 0) {
L
Linus Torvalds 已提交
814 815
			ip_rt_put(rt);
			ret = NULL;
816 817
		} else if ((rt->rt_flags & RTCF_REDIRECTED) ||
			   rt->dst.expires) {
D
David S. Miller 已提交
818
			ip_rt_put(rt);
L
Linus Torvalds 已提交
819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
			ret = NULL;
		}
	}
	return ret;
}

/*
 * Algorithm:
 *	1. The first ip_rt_redirect_number redirects are sent
 *	   with exponential backoff, then we stop sending them at all,
 *	   assuming that the host ignores our redirects.
 *	2. If we did not see packets requiring redirects
 *	   during ip_rt_redirect_silence, we assume that the host
 *	   forgot redirected route and start to send redirects again.
 *
 * This algorithm is much cheaper and more intelligent than dumb load limiting
 * in icmp.c.
 *
 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 * and "frag. need" (breaks PMTU discovery) in icmp.c.
 */

void ip_rt_send_redirect(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
843
	struct rtable *rt = skb_rtable(skb);
844
	struct in_device *in_dev;
845
	struct inet_peer *peer;
846
	struct net *net;
847
	int log_martians;
848
	int vif;
L
Linus Torvalds 已提交
849

850
	rcu_read_lock();
851
	in_dev = __in_dev_get_rcu(rt->dst.dev);
852 853
	if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
		rcu_read_unlock();
L
Linus Torvalds 已提交
854
		return;
855 856
	}
	log_martians = IN_DEV_LOG_MARTIANS(in_dev);
857
	vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
858
	rcu_read_unlock();
L
Linus Torvalds 已提交
859

860
	net = dev_net(rt->dst.dev);
861
	peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
862
	if (!peer) {
J
Julian Anastasov 已提交
863 864
		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
			  rt_nexthop(rt, ip_hdr(skb)->daddr));
865 866 867
		return;
	}

L
Linus Torvalds 已提交
868 869 870
	/* No redirected packets during ip_rt_redirect_silence;
	 * reset the algorithm.
	 */
871 872
	if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
		peer->rate_tokens = 0;
L
Linus Torvalds 已提交
873 874

	/* Too many ignored redirects; do not send anything
875
	 * set dst.rate_last to the last seen redirected packet.
L
Linus Torvalds 已提交
876
	 */
877 878
	if (peer->rate_tokens >= ip_rt_redirect_number) {
		peer->rate_last = jiffies;
879
		goto out_put_peer;
L
Linus Torvalds 已提交
880 881 882 883 884
	}

	/* Check for load limit; set rate_last to the latest sent
	 * redirect.
	 */
885
	if (peer->rate_tokens == 0 ||
886
	    time_after(jiffies,
887 888
		       (peer->rate_last +
			(ip_rt_redirect_load << peer->rate_tokens)))) {
J
Julian Anastasov 已提交
889 890 891
		__be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);

		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
892 893
		peer->rate_last = jiffies;
		++peer->rate_tokens;
L
Linus Torvalds 已提交
894
#ifdef CONFIG_IP_ROUTE_VERBOSE
895
		if (log_martians &&
896 897
		    peer->rate_tokens == ip_rt_redirect_number)
			net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
898
					     &ip_hdr(skb)->saddr, inet_iif(skb),
J
Julian Anastasov 已提交
899
					     &ip_hdr(skb)->daddr, &gw);
L
Linus Torvalds 已提交
900 901
#endif
	}
902 903
out_put_peer:
	inet_putpeer(peer);
L
Linus Torvalds 已提交
904 905 906 907
}

static int ip_error(struct sk_buff *skb)
{
908
	struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
E
Eric Dumazet 已提交
909
	struct rtable *rt = skb_rtable(skb);
910
	struct inet_peer *peer;
L
Linus Torvalds 已提交
911
	unsigned long now;
912
	struct net *net;
913
	bool send;
L
Linus Torvalds 已提交
914 915
	int code;

916 917 918 919
	/* IP on this device is disabled. */
	if (!in_dev)
		goto out;

920 921 922 923
	net = dev_net(rt->dst.dev);
	if (!IN_DEV_FORWARD(in_dev)) {
		switch (rt->dst.error) {
		case EHOSTUNREACH:
E
Eric Dumazet 已提交
924
			__IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
925 926 927
			break;

		case ENETUNREACH:
E
Eric Dumazet 已提交
928
			__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
929 930 931 932 933
			break;
		}
		goto out;
	}

934
	switch (rt->dst.error) {
J
Joe Perches 已提交
935 936 937 938 939 940 941 942
	case EINVAL:
	default:
		goto out;
	case EHOSTUNREACH:
		code = ICMP_HOST_UNREACH;
		break;
	case ENETUNREACH:
		code = ICMP_NET_UNREACH;
E
Eric Dumazet 已提交
943
		__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
J
Joe Perches 已提交
944 945 946 947
		break;
	case EACCES:
		code = ICMP_PKT_FILTERED;
		break;
L
Linus Torvalds 已提交
948 949
	}

950
	peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
951
			       l3mdev_master_ifindex(skb->dev), 1);
952 953 954 955 956 957 958 959 960 961 962 963

	send = true;
	if (peer) {
		now = jiffies;
		peer->rate_tokens += now - peer->rate_last;
		if (peer->rate_tokens > ip_rt_error_burst)
			peer->rate_tokens = ip_rt_error_burst;
		peer->rate_last = now;
		if (peer->rate_tokens >= ip_rt_error_cost)
			peer->rate_tokens -= ip_rt_error_cost;
		else
			send = false;
964
		inet_putpeer(peer);
L
Linus Torvalds 已提交
965
	}
966 967
	if (send)
		icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
L
Linus Torvalds 已提交
968 969 970

out:	kfree_skb(skb);
	return 0;
971
}
L
Linus Torvalds 已提交
972

973
static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
L
Linus Torvalds 已提交
974
{
975
	struct dst_entry *dst = &rt->dst;
976
	struct fib_result res;
977

978 979 980
	if (dst_metric_locked(dst, RTAX_MTU))
		return;

981
	if (ipv4_mtu(dst) < mtu)
982 983
		return;

984 985
	if (mtu < ip_rt_min_pmtu)
		mtu = ip_rt_min_pmtu;
986

987 988 989 990
	if (rt->rt_pmtu == mtu &&
	    time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
		return;

991
	rcu_read_lock();
992
	if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
993 994
		struct fib_nh *nh = &FIB_RES_NH(res);

995 996
		update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
				      jiffies + ip_rt_mtu_expires);
997
	}
998
	rcu_read_unlock();
L
Linus Torvalds 已提交
999 1000
}

1001 1002 1003 1004 1005 1006 1007
static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
			      struct sk_buff *skb, u32 mtu)
{
	struct rtable *rt = (struct rtable *) dst;
	struct flowi4 fl4;

	ip_rt_build_flow_key(&fl4, sk, skb);
1008
	__ip_rt_update_pmtu(rt, &fl4, mtu);
1009 1010
}

1011 1012 1013
void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
		      int oif, u32 mark, u8 protocol, int flow_flags)
{
1014
	const struct iphdr *iph = (const struct iphdr *) skb->data;
1015 1016 1017
	struct flowi4 fl4;
	struct rtable *rt;

1018 1019 1020
	if (!mark)
		mark = IP4_REPLY_MARK(net, skb->mark);

1021 1022
	__build_flow_key(&fl4, NULL, iph, oif,
			 RT_TOS(iph->tos), protocol, mark, flow_flags);
1023 1024
	rt = __ip_route_output_key(net, &fl4);
	if (!IS_ERR(rt)) {
1025
		__ip_rt_update_pmtu(rt, &fl4, mtu);
1026 1027 1028 1029 1030
		ip_rt_put(rt);
	}
}
EXPORT_SYMBOL_GPL(ipv4_update_pmtu);

1031
static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1032
{
1033 1034 1035
	const struct iphdr *iph = (const struct iphdr *) skb->data;
	struct flowi4 fl4;
	struct rtable *rt;
1036

1037
	__build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1038 1039 1040 1041

	if (!fl4.flowi4_mark)
		fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);

1042 1043 1044 1045 1046
	rt = __ip_route_output_key(sock_net(sk), &fl4);
	if (!IS_ERR(rt)) {
		__ip_rt_update_pmtu(rt, &fl4, mtu);
		ip_rt_put(rt);
	}
1047
}
1048 1049 1050 1051 1052 1053

void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
{
	const struct iphdr *iph = (const struct iphdr *) skb->data;
	struct flowi4 fl4;
	struct rtable *rt;
1054
	struct dst_entry *odst = NULL;
1055
	bool new = false;
1056 1057

	bh_lock_sock(sk);
1058 1059 1060 1061

	if (!ip_sk_accept_pmtu(sk))
		goto out;

1062
	odst = sk_dst_get(sk);
1063

1064
	if (sock_owned_by_user(sk) || !odst) {
1065 1066 1067 1068 1069 1070
		__ipv4_sk_update_pmtu(skb, sk, mtu);
		goto out;
	}

	__build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);

1071
	rt = (struct rtable *)odst;
1072
	if (odst->obsolete && !odst->ops->check(odst, 0)) {
1073 1074 1075
		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
		if (IS_ERR(rt))
			goto out;
1076 1077

		new = true;
1078 1079 1080 1081
	}

	__ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);

1082
	if (!dst_check(&rt->dst, 0)) {
1083 1084 1085
		if (new)
			dst_release(&rt->dst);

1086 1087 1088 1089
		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
		if (IS_ERR(rt))
			goto out;

1090
		new = true;
1091 1092
	}

1093
	if (new)
1094
		sk_dst_set(sk, &rt->dst);
1095 1096 1097

out:
	bh_unlock_sock(sk);
1098
	dst_release(odst);
1099
}
1100
EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1101

1102 1103 1104
void ipv4_redirect(struct sk_buff *skb, struct net *net,
		   int oif, u32 mark, u8 protocol, int flow_flags)
{
1105
	const struct iphdr *iph = (const struct iphdr *) skb->data;
1106 1107 1108
	struct flowi4 fl4;
	struct rtable *rt;

1109 1110
	__build_flow_key(&fl4, NULL, iph, oif,
			 RT_TOS(iph->tos), protocol, mark, flow_flags);
1111 1112
	rt = __ip_route_output_key(net, &fl4);
	if (!IS_ERR(rt)) {
1113
		__ip_do_redirect(rt, skb, &fl4, false);
1114 1115 1116 1117 1118 1119 1120
		ip_rt_put(rt);
	}
}
EXPORT_SYMBOL_GPL(ipv4_redirect);

void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
{
1121 1122 1123
	const struct iphdr *iph = (const struct iphdr *) skb->data;
	struct flowi4 fl4;
	struct rtable *rt;
1124

1125 1126 1127
	__build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
	rt = __ip_route_output_key(sock_net(sk), &fl4);
	if (!IS_ERR(rt)) {
1128
		__ip_do_redirect(rt, skb, &fl4, false);
1129 1130
		ip_rt_put(rt);
	}
1131 1132 1133
}
EXPORT_SYMBOL_GPL(ipv4_sk_redirect);

1134 1135 1136 1137
static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
{
	struct rtable *rt = (struct rtable *) dst;

1138 1139 1140 1141
	/* All IPV4 dsts are created with ->obsolete set to the value
	 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
	 * into this function always.
	 *
1142 1143 1144
	 * When a PMTU/redirect information update invalidates a route,
	 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
	 * DST_OBSOLETE_DEAD by dst_free().
1145
	 */
1146
	if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1147
		return NULL;
T
Timo Teräs 已提交
1148
	return dst;
L
Linus Torvalds 已提交
1149 1150 1151 1152 1153 1154 1155 1156
}

static void ipv4_link_failure(struct sk_buff *skb)
{
	struct rtable *rt;

	icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);

E
Eric Dumazet 已提交
1157
	rt = skb_rtable(skb);
1158 1159
	if (rt)
		dst_set_expires(&rt->dst, 0);
L
Linus Torvalds 已提交
1160 1161
}

E
Eric W. Biederman 已提交
1162
static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
1163
{
1164 1165 1166
	pr_debug("%s: %pI4 -> %pI4, %s\n",
		 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
		 skb->dev ? skb->dev->name : "?");
L
Linus Torvalds 已提交
1167
	kfree_skb(skb);
1168
	WARN_ON(1);
L
Linus Torvalds 已提交
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	return 0;
}

/*
   We do not cache source address of outgoing interface,
   because it is used only by IP RR, TS and SRR options,
   so that it out of fast path.

   BTW remember: "addr" is allowed to be not aligned
   in IP options!
 */

1181
void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
L
Linus Torvalds 已提交
1182
{
A
Al Viro 已提交
1183
	__be32 src;
L
Linus Torvalds 已提交
1184

1185
	if (rt_is_output_route(rt))
1186
		src = ip_hdr(skb)->saddr;
E
Eric Dumazet 已提交
1187
	else {
1188 1189 1190 1191 1192 1193 1194 1195 1196
		struct fib_result res;
		struct flowi4 fl4;
		struct iphdr *iph;

		iph = ip_hdr(skb);

		memset(&fl4, 0, sizeof(fl4));
		fl4.daddr = iph->daddr;
		fl4.saddr = iph->saddr;
1197
		fl4.flowi4_tos = RT_TOS(iph->tos);
1198 1199 1200
		fl4.flowi4_oif = rt->dst.dev->ifindex;
		fl4.flowi4_iif = skb->dev->ifindex;
		fl4.flowi4_mark = skb->mark;
1201

E
Eric Dumazet 已提交
1202
		rcu_read_lock();
1203
		if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1204
			src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
E
Eric Dumazet 已提交
1205
		else
1206 1207 1208
			src = inet_select_addr(rt->dst.dev,
					       rt_nexthop(rt, iph->daddr),
					       RT_SCOPE_UNIVERSE);
E
Eric Dumazet 已提交
1209 1210
		rcu_read_unlock();
	}
L
Linus Torvalds 已提交
1211 1212 1213
	memcpy(addr, &src, 4);
}

1214
#ifdef CONFIG_IP_ROUTE_CLASSID
L
Linus Torvalds 已提交
1215 1216
static void set_class_tag(struct rtable *rt, u32 tag)
{
1217 1218 1219 1220
	if (!(rt->dst.tclassid & 0xFFFF))
		rt->dst.tclassid |= tag & 0xFFFF;
	if (!(rt->dst.tclassid & 0xFFFF0000))
		rt->dst.tclassid |= tag & 0xFFFF0000;
L
Linus Torvalds 已提交
1221 1222 1223
}
#endif

1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
{
	unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);

	if (advmss == 0) {
		advmss = max_t(unsigned int, dst->dev->mtu - 40,
			       ip_rt_min_advmss);
		if (advmss > 65535 - 40)
			advmss = 65535 - 40;
	}
	return advmss;
}

1237
static unsigned int ipv4_mtu(const struct dst_entry *dst)
1238
{
1239
	const struct rtable *rt = (const struct rtable *) dst;
1240 1241
	unsigned int mtu = rt->rt_pmtu;

1242
	if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1243
		mtu = dst_metric_raw(dst, RTAX_MTU);
1244

1245
	if (mtu)
1246 1247 1248
		return mtu;

	mtu = dst->dev->mtu;
1249 1250

	if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
J
Julian Anastasov 已提交
1251
		if (rt->rt_uses_gateway && mtu > 576)
1252 1253 1254
			mtu = 576;
	}

1255 1256 1257
	mtu = min_t(unsigned int, mtu, IP_MAX_MTU);

	return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1258 1259
}

1260
static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1261
{
1262
	struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1263 1264 1265
	struct fib_nh_exception *fnhe;
	u32 hval;

1266 1267 1268
	if (!hash)
		return NULL;

1269
	hval = fnhe_hashfun(daddr);
1270 1271 1272

	for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
1273 1274 1275 1276 1277
		if (fnhe->fnhe_daddr == daddr)
			return fnhe;
	}
	return NULL;
}
1278

1279
static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1280 1281
			      __be32 daddr)
{
1282 1283
	bool ret = false;

1284
	spin_lock_bh(&fnhe_lock);
1285

1286
	if (daddr == fnhe->fnhe_daddr) {
1287 1288
		struct rtable __rcu **porig;
		struct rtable *orig;
1289
		int genid = fnhe_genid(dev_net(rt->dst.dev));
1290 1291 1292 1293 1294 1295

		if (rt_is_input_route(rt))
			porig = &fnhe->fnhe_rth_input;
		else
			porig = &fnhe->fnhe_rth_output;
		orig = rcu_dereference(*porig);
1296 1297 1298

		if (fnhe->fnhe_genid != genid) {
			fnhe->fnhe_genid = genid;
1299 1300 1301
			fnhe->fnhe_gw = 0;
			fnhe->fnhe_pmtu = 0;
			fnhe->fnhe_expires = 0;
1302 1303
			fnhe_flush_routes(fnhe);
			orig = NULL;
1304
		}
1305 1306
		fill_route_from_fnhe(rt, fnhe);
		if (!rt->rt_gateway)
J
Julian Anastasov 已提交
1307
			rt->rt_gateway = daddr;
1308

1309 1310 1311 1312 1313 1314
		if (!(rt->dst.flags & DST_NOCACHE)) {
			rcu_assign_pointer(*porig, rt);
			if (orig)
				rt_free(orig);
			ret = true;
		}
1315 1316 1317 1318

		fnhe->fnhe_stamp = jiffies;
	}
	spin_unlock_bh(&fnhe_lock);
1319 1320

	return ret;
1321 1322
}

1323
static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1324
{
E
Eric Dumazet 已提交
1325
	struct rtable *orig, *prev, **p;
1326
	bool ret = true;
1327

E
Eric Dumazet 已提交
1328
	if (rt_is_input_route(rt)) {
1329
		p = (struct rtable **)&nh->nh_rth_input;
E
Eric Dumazet 已提交
1330
	} else {
1331
		p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
E
Eric Dumazet 已提交
1332
	}
1333 1334 1335 1336 1337
	orig = *p;

	prev = cmpxchg(p, orig, rt);
	if (prev == orig) {
		if (orig)
1338
			rt_free(orig);
J
Julian Anastasov 已提交
1339
	} else
1340 1341 1342 1343 1344
		ret = false;

	return ret;
}

E
Eric Dumazet 已提交
1345 1346 1347 1348 1349 1350
struct uncached_list {
	spinlock_t		lock;
	struct list_head	head;
};

static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1351 1352 1353

static void rt_add_uncached_list(struct rtable *rt)
{
E
Eric Dumazet 已提交
1354 1355 1356 1357 1358 1359 1360
	struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);

	rt->rt_uncached_list = ul;

	spin_lock_bh(&ul->lock);
	list_add_tail(&rt->rt_uncached, &ul->head);
	spin_unlock_bh(&ul->lock);
1361 1362 1363 1364 1365 1366
}

static void ipv4_dst_destroy(struct dst_entry *dst)
{
	struct rtable *rt = (struct rtable *) dst;

1367
	if (!list_empty(&rt->rt_uncached)) {
E
Eric Dumazet 已提交
1368 1369 1370
		struct uncached_list *ul = rt->rt_uncached_list;

		spin_lock_bh(&ul->lock);
1371
		list_del(&rt->rt_uncached);
E
Eric Dumazet 已提交
1372
		spin_unlock_bh(&ul->lock);
1373 1374 1375 1376 1377
	}
}

void rt_flush_dev(struct net_device *dev)
{
E
Eric Dumazet 已提交
1378 1379 1380 1381 1382 1383
	struct net *net = dev_net(dev);
	struct rtable *rt;
	int cpu;

	for_each_possible_cpu(cpu) {
		struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1384

E
Eric Dumazet 已提交
1385 1386
		spin_lock_bh(&ul->lock);
		list_for_each_entry(rt, &ul->head, rt_uncached) {
1387 1388 1389 1390 1391 1392
			if (rt->dst.dev != dev)
				continue;
			rt->dst.dev = net->loopback_dev;
			dev_hold(rt->dst.dev);
			dev_put(dev);
		}
E
Eric Dumazet 已提交
1393
		spin_unlock_bh(&ul->lock);
1394 1395 1396
	}
}

1397
static bool rt_cache_valid(const struct rtable *rt)
1398
{
1399 1400 1401
	return	rt &&
		rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
		!rt_is_expired(rt);
1402 1403
}

1404
static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1405
			   const struct fib_result *res,
1406
			   struct fib_nh_exception *fnhe,
1407
			   struct fib_info *fi, u16 type, u32 itag)
L
Linus Torvalds 已提交
1408
{
1409 1410
	bool cached = false;

L
Linus Torvalds 已提交
1411
	if (fi) {
1412 1413
		struct fib_nh *nh = &FIB_RES_NH(*res);

J
Julian Anastasov 已提交
1414
		if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1415
			rt->rt_gateway = nh->nh_gw;
J
Julian Anastasov 已提交
1416 1417
			rt->rt_uses_gateway = 1;
		}
D
David S. Miller 已提交
1418
		dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1419
#ifdef CONFIG_IP_ROUTE_CLASSID
1420
		rt->dst.tclassid = nh->nh_tclassid;
L
Linus Torvalds 已提交
1421
#endif
1422
		rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1423
		if (unlikely(fnhe))
1424
			cached = rt_bind_exception(rt, fnhe, daddr);
1425
		else if (!(rt->dst.flags & DST_NOCACHE))
1426
			cached = rt_cache_route(nh, rt);
J
Julian Anastasov 已提交
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
		if (unlikely(!cached)) {
			/* Routes we intend to cache in nexthop exception or
			 * FIB nexthop have the DST_NOCACHE bit clear.
			 * However, if we are unsuccessful at storing this
			 * route into the cache we really need to set it.
			 */
			rt->dst.flags |= DST_NOCACHE;
			if (!rt->rt_gateway)
				rt->rt_gateway = daddr;
			rt_add_uncached_list(rt);
		}
	} else
1439
		rt_add_uncached_list(rt);
1440

1441
#ifdef CONFIG_IP_ROUTE_CLASSID
L
Linus Torvalds 已提交
1442
#ifdef CONFIG_IP_MULTIPLE_TABLES
1443
	set_class_tag(rt, res->tclassid);
L
Linus Torvalds 已提交
1444 1445 1446 1447 1448
#endif
	set_class_tag(rt, itag);
#endif
}

1449 1450 1451
struct rtable *rt_dst_alloc(struct net_device *dev,
			    unsigned int flags, u16 type,
			    bool nopolicy, bool noxfrm, bool will_cache)
1452
{
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468
	struct rtable *rt;

	rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
		       (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
		       (nopolicy ? DST_NOPOLICY : 0) |
		       (noxfrm ? DST_NOXFRM : 0));

	if (rt) {
		rt->rt_genid = rt_genid_ipv4(dev_net(dev));
		rt->rt_flags = flags;
		rt->rt_type = type;
		rt->rt_is_input = 0;
		rt->rt_iif = 0;
		rt->rt_pmtu = 0;
		rt->rt_gateway = 0;
		rt->rt_uses_gateway = 0;
D
David Ahern 已提交
1469
		rt->rt_table_id = 0;
1470 1471 1472 1473 1474 1475 1476 1477
		INIT_LIST_HEAD(&rt->rt_uncached);

		rt->dst.output = ip_output;
		if (flags & RTCF_LOCAL)
			rt->dst.input = ip_local_deliver;
	}

	return rt;
1478
}
1479
EXPORT_SYMBOL(rt_dst_alloc);
1480

1481
/* called in rcu_read_lock() section */
A
Al Viro 已提交
1482
static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
L
Linus Torvalds 已提交
1483 1484 1485
				u8 tos, struct net_device *dev, int our)
{
	struct rtable *rth;
1486
	struct in_device *in_dev = __in_dev_get_rcu(dev);
1487
	unsigned int flags = RTCF_MULTICAST;
L
Linus Torvalds 已提交
1488
	u32 itag = 0;
1489
	int err;
L
Linus Torvalds 已提交
1490 1491 1492

	/* Primary sanity checks. */

1493
	if (!in_dev)
L
Linus Torvalds 已提交
1494 1495
		return -EINVAL;

1496
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1497
	    skb->protocol != htons(ETH_P_IP))
L
Linus Torvalds 已提交
1498 1499
		goto e_inval;

1500 1501
	if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
		goto e_inval;
1502

1503 1504
	if (ipv4_is_zeronet(saddr)) {
		if (!ipv4_is_local_multicast(daddr))
L
Linus Torvalds 已提交
1505
			goto e_inval;
1506
	} else {
1507 1508
		err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
					  in_dev, &itag);
1509 1510 1511
		if (err < 0)
			goto e_err;
	}
1512 1513 1514 1515
	if (our)
		flags |= RTCF_LOCAL;

	rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1516
			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
L
Linus Torvalds 已提交
1517 1518 1519
	if (!rth)
		goto e_nobufs;

1520 1521 1522
#ifdef CONFIG_IP_ROUTE_CLASSID
	rth->dst.tclassid = itag;
#endif
1523
	rth->dst.output = ip_rt_bug;
1524
	rth->rt_is_input= 1;
L
Linus Torvalds 已提交
1525 1526

#ifdef CONFIG_IP_MROUTE
1527
	if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1528
		rth->dst.input = ip_mr_input;
L
Linus Torvalds 已提交
1529 1530 1531
#endif
	RT_CACHE_STAT_INC(in_slow_mc);

D
David S. Miller 已提交
1532 1533
	skb_dst_set(skb, &rth->dst);
	return 0;
L
Linus Torvalds 已提交
1534 1535 1536 1537

e_nobufs:
	return -ENOBUFS;
e_inval:
1538
	return -EINVAL;
1539 1540
e_err:
	return err;
L
Linus Torvalds 已提交
1541 1542 1543 1544 1545 1546
}


static void ip_handle_martian_source(struct net_device *dev,
				     struct in_device *in_dev,
				     struct sk_buff *skb,
A
Al Viro 已提交
1547 1548
				     __be32 daddr,
				     __be32 saddr)
L
Linus Torvalds 已提交
1549 1550 1551 1552 1553 1554 1555 1556
{
	RT_CACHE_STAT_INC(in_martian_src);
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
		/*
		 *	RFC1812 recommendation, if source is martian,
		 *	the only hint is MAC header.
		 */
J
Joe Perches 已提交
1557
		pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1558
			&daddr, &saddr, dev->name);
1559
		if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
J
Joe Perches 已提交
1560 1561 1562 1563
			print_hex_dump(KERN_WARNING, "ll header: ",
				       DUMP_PREFIX_OFFSET, 16, 1,
				       skb_mac_header(skb),
				       dev->hard_header_len, true);
L
Linus Torvalds 已提交
1564 1565 1566 1567 1568
		}
	}
#endif
}

1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
{
	struct fnhe_hash_bucket *hash;
	struct fib_nh_exception *fnhe, __rcu **fnhe_p;
	u32 hval = fnhe_hashfun(daddr);

	spin_lock_bh(&fnhe_lock);

	hash = rcu_dereference_protected(nh->nh_exceptions,
					 lockdep_is_held(&fnhe_lock));
	hash += hval;

	fnhe_p = &hash->chain;
	fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
	while (fnhe) {
		if (fnhe->fnhe_daddr == daddr) {
			rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
				fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
			fnhe_flush_routes(fnhe);
			kfree_rcu(fnhe, rcu);
			break;
		}
		fnhe_p = &fnhe->fnhe_next;
		fnhe = rcu_dereference_protected(fnhe->fnhe_next,
						 lockdep_is_held(&fnhe_lock));
	}

	spin_unlock_bh(&fnhe_lock);
}

1599
/* called in rcu_read_lock() section */
S
Stephen Hemminger 已提交
1600
static int __mkroute_input(struct sk_buff *skb,
1601
			   const struct fib_result *res,
S
Stephen Hemminger 已提交
1602
			   struct in_device *in_dev,
1603
			   __be32 daddr, __be32 saddr, u32 tos)
L
Linus Torvalds 已提交
1604
{
1605
	struct fib_nh_exception *fnhe;
L
Linus Torvalds 已提交
1606 1607 1608
	struct rtable *rth;
	int err;
	struct in_device *out_dev;
1609
	bool do_cache;
1610
	u32 itag = 0;
L
Linus Torvalds 已提交
1611 1612

	/* get a working reference to the output device */
1613
	out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1614
	if (!out_dev) {
1615
		net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
L
Linus Torvalds 已提交
1616 1617 1618
		return -EINVAL;
	}

1619
	err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1620
				  in_dev->dev, in_dev, &itag);
L
Linus Torvalds 已提交
1621
	if (err < 0) {
1622
		ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
L
Linus Torvalds 已提交
1623
					 saddr);
1624

L
Linus Torvalds 已提交
1625 1626 1627
		goto cleanup;
	}

J
Julian Anastasov 已提交
1628 1629
	do_cache = res->fi && !itag;
	if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1630
	    skb->protocol == htons(ETH_P_IP) &&
L
Linus Torvalds 已提交
1631
	    (IN_DEV_SHARED_MEDIA(out_dev) ||
1632 1633
	     inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
		IPCB(skb)->flags |= IPSKB_DOREDIRECT;
L
Linus Torvalds 已提交
1634 1635 1636 1637

	if (skb->protocol != htons(ETH_P_IP)) {
		/* Not IP (i.e. ARP). Do not create route, if it is
		 * invalid for proxy arp. DNAT routes are always valid.
1638 1639 1640 1641
		 *
		 * Proxy arp feature have been extended to allow, ARP
		 * replies back to the same interface, to support
		 * Private VLAN switch technologies. See arp.c.
L
Linus Torvalds 已提交
1642
		 */
1643 1644
		if (out_dev == in_dev &&
		    IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
L
Linus Torvalds 已提交
1645 1646 1647 1648 1649
			err = -EINVAL;
			goto cleanup;
		}
	}

1650
	fnhe = find_exception(&FIB_RES_NH(*res), daddr);
J
Julian Anastasov 已提交
1651
	if (do_cache) {
1652
		if (fnhe) {
1653
			rth = rcu_dereference(fnhe->fnhe_rth_input);
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
			if (rth && rth->dst.expires &&
			    time_after(jiffies, rth->dst.expires)) {
				ip_del_fnhe(&FIB_RES_NH(*res), daddr);
				fnhe = NULL;
			} else {
				goto rt_cache;
			}
		}

		rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1664

1665
rt_cache:
J
Julian Anastasov 已提交
1666 1667 1668
		if (rt_cache_valid(rth)) {
			skb_dst_set_noref(skb, &rth->dst);
			goto out;
1669 1670
		}
	}
1671

1672
	rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1673
			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
1674
			   IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
L
Linus Torvalds 已提交
1675 1676 1677 1678 1679
	if (!rth) {
		err = -ENOBUFS;
		goto cleanup;
	}

1680
	rth->rt_is_input = 1;
D
David Ahern 已提交
1681 1682
	if (res->table)
		rth->rt_table_id = res->table->tb_id;
D
Duan Jiong 已提交
1683
	RT_CACHE_STAT_INC(in_slow_tot);
L
Linus Torvalds 已提交
1684

1685
	rth->dst.input = ip_forward;
L
Linus Torvalds 已提交
1686

1687
	rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag);
1688 1689
	if (lwtunnel_output_redirect(rth->dst.lwtstate)) {
		rth->dst.lwtstate->orig_output = rth->dst.output;
1690
		rth->dst.output = lwtunnel_output;
1691
	}
1692 1693
	if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
		rth->dst.lwtstate->orig_input = rth->dst.input;
1694 1695
		rth->dst.input = lwtunnel_input;
	}
1696
	skb_dst_set(skb, &rth->dst);
1697
out:
L
Linus Torvalds 已提交
1698 1699 1700
	err = 0;
 cleanup:
	return err;
1701
}
L
Linus Torvalds 已提交
1702

1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
#ifdef CONFIG_IP_ROUTE_MULTIPATH

/* To make ICMP packets follow the right flow, the multipath hash is
 * calculated from the inner IP addresses in reverse order.
 */
static int ip_multipath_icmp_hash(struct sk_buff *skb)
{
	const struct iphdr *outer_iph = ip_hdr(skb);
	struct icmphdr _icmph;
	const struct icmphdr *icmph;
	struct iphdr _inner_iph;
	const struct iphdr *inner_iph;

	if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
		goto standard_hash;

	icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
				   &_icmph);
	if (!icmph)
		goto standard_hash;

	if (icmph->type != ICMP_DEST_UNREACH &&
	    icmph->type != ICMP_REDIRECT &&
	    icmph->type != ICMP_TIME_EXCEEDED &&
	    icmph->type != ICMP_PARAMETERPROB) {
		goto standard_hash;
	}

	inner_iph = skb_header_pointer(skb,
				       outer_iph->ihl * 4 + sizeof(_icmph),
				       sizeof(_inner_iph), &_inner_iph);
	if (!inner_iph)
		goto standard_hash;

	return fib_multipath_hash(inner_iph->daddr, inner_iph->saddr);

standard_hash:
	return fib_multipath_hash(outer_iph->saddr, outer_iph->daddr);
}

#endif /* CONFIG_IP_ROUTE_MULTIPATH */

S
Stephen Hemminger 已提交
1745 1746
static int ip_mkroute_input(struct sk_buff *skb,
			    struct fib_result *res,
1747
			    const struct flowi4 *fl4,
S
Stephen Hemminger 已提交
1748 1749
			    struct in_device *in_dev,
			    __be32 daddr, __be32 saddr, u32 tos)
L
Linus Torvalds 已提交
1750 1751
{
#ifdef CONFIG_IP_ROUTE_MULTIPATH
P
Peter Nørlund 已提交
1752 1753 1754
	if (res->fi && res->fi->fib_nhs > 1) {
		int h;

1755 1756 1757 1758
		if (unlikely(ip_hdr(skb)->protocol == IPPROTO_ICMP))
			h = ip_multipath_icmp_hash(skb);
		else
			h = fib_multipath_hash(saddr, daddr);
P
Peter Nørlund 已提交
1759 1760
		fib_select_multipath(res, h);
	}
L
Linus Torvalds 已提交
1761 1762 1763
#endif

	/* create a routing cache entry */
1764
	return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
L
Linus Torvalds 已提交
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
}

/*
 *	NOTE. We drop all the packets that has local source
 *	addresses, because every properly looped back packet
 *	must have correct destination already attached by output routine.
 *
 *	Such approach solves two big problems:
 *	1. Not simplex devices are handled properly.
 *	2. IP spoofing attempts are filtered with 100% of guarantee.
E
Eric Dumazet 已提交
1775
 *	called with rcu_read_lock()
L
Linus Torvalds 已提交
1776 1777
 */

A
Al Viro 已提交
1778
static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1779
			       u8 tos, struct net_device *dev)
L
Linus Torvalds 已提交
1780 1781
{
	struct fib_result res;
1782
	struct in_device *in_dev = __in_dev_get_rcu(dev);
1783
	struct ip_tunnel_info *tun_info;
1784
	struct flowi4	fl4;
1785
	unsigned int	flags = 0;
L
Linus Torvalds 已提交
1786
	u32		itag = 0;
1787
	struct rtable	*rth;
L
Linus Torvalds 已提交
1788
	int		err = -EINVAL;
D
Daniel Baluta 已提交
1789
	struct net    *net = dev_net(dev);
1790
	bool do_cache;
L
Linus Torvalds 已提交
1791 1792 1793 1794 1795 1796 1797 1798 1799 1800

	/* IP on this device is disabled. */

	if (!in_dev)
		goto out;

	/* Check for the most weird martians, which can be not detected
	   by fib_lookup.
	 */

1801
	tun_info = skb_tunnel_info(skb);
1802
	if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1803 1804 1805
		fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
	else
		fl4.flowi4_tun_key.tun_id = 0;
T
Thomas Graf 已提交
1806 1807
	skb_dst_drop(skb);

1808
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
L
Linus Torvalds 已提交
1809 1810
		goto martian_source;

1811
	res.fi = NULL;
1812
	res.table = NULL;
1813
	if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
L
Linus Torvalds 已提交
1814 1815 1816 1817 1818
		goto brd_input;

	/* Accept zero addresses only to limited broadcast;
	 * I even do not know to fix it or not. Waiting for complains :-)
	 */
1819
	if (ipv4_is_zeronet(saddr))
L
Linus Torvalds 已提交
1820 1821
		goto martian_source;

1822
	if (ipv4_is_zeronet(daddr))
L
Linus Torvalds 已提交
1823 1824
		goto martian_destination;

1825 1826 1827 1828 1829
	/* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
	 * and call it once if daddr or/and saddr are loopback addresses
	 */
	if (ipv4_is_loopback(daddr)) {
		if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1830
			goto martian_destination;
1831 1832
	} else if (ipv4_is_loopback(saddr)) {
		if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1833 1834 1835
			goto martian_source;
	}

L
Linus Torvalds 已提交
1836 1837 1838
	/*
	 *	Now we are ready to route packet.
	 */
1839
	fl4.flowi4_oif = 0;
1840
	fl4.flowi4_iif = dev->ifindex;
1841 1842 1843
	fl4.flowi4_mark = skb->mark;
	fl4.flowi4_tos = tos;
	fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1844
	fl4.flowi4_flags = 0;
1845 1846
	fl4.daddr = daddr;
	fl4.saddr = saddr;
1847
	err = fib_lookup(net, &fl4, &res, 0);
1848 1849 1850
	if (err != 0) {
		if (!IN_DEV_FORWARD(in_dev))
			err = -EHOSTUNREACH;
L
Linus Torvalds 已提交
1851
		goto no_route;
1852
	}
L
Linus Torvalds 已提交
1853 1854 1855 1856 1857

	if (res.type == RTN_BROADCAST)
		goto brd_input;

	if (res.type == RTN_LOCAL) {
1858
		err = fib_validate_source(skb, saddr, daddr, tos,
1859
					  0, dev, in_dev, &itag);
1860
		if (err < 0)
1861
			goto martian_source;
L
Linus Torvalds 已提交
1862 1863 1864
		goto local_input;
	}

1865 1866
	if (!IN_DEV_FORWARD(in_dev)) {
		err = -EHOSTUNREACH;
1867
		goto no_route;
1868
	}
L
Linus Torvalds 已提交
1869 1870 1871
	if (res.type != RTN_UNICAST)
		goto martian_destination;

1872
	err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
L
Linus Torvalds 已提交
1873 1874 1875 1876 1877 1878
out:	return err;

brd_input:
	if (skb->protocol != htons(ETH_P_IP))
		goto e_inval;

1879
	if (!ipv4_is_zeronet(saddr)) {
1880 1881
		err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
					  in_dev, &itag);
L
Linus Torvalds 已提交
1882
		if (err < 0)
1883
			goto martian_source;
L
Linus Torvalds 已提交
1884 1885 1886 1887 1888 1889
	}
	flags |= RTCF_BROADCAST;
	res.type = RTN_BROADCAST;
	RT_CACHE_STAT_INC(in_brd);

local_input:
1890 1891
	do_cache = false;
	if (res.fi) {
1892
		if (!itag) {
1893
			rth = rcu_dereference(FIB_RES_NH(res).nh_rth_input);
1894
			if (rt_cache_valid(rth)) {
1895 1896 1897
				skb_dst_set_noref(skb, &rth->dst);
				err = 0;
				goto out;
1898 1899 1900 1901 1902
			}
			do_cache = true;
		}
	}

1903
	rth = rt_dst_alloc(net->loopback_dev, flags | RTCF_LOCAL, res.type,
1904
			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
L
Linus Torvalds 已提交
1905 1906 1907
	if (!rth)
		goto e_nobufs;

1908
	rth->dst.output= ip_rt_bug;
1909 1910 1911
#ifdef CONFIG_IP_ROUTE_CLASSID
	rth->dst.tclassid = itag;
#endif
1912
	rth->rt_is_input = 1;
D
David Ahern 已提交
1913 1914
	if (res.table)
		rth->rt_table_id = res.table->tb_id;
1915

D
Duan Jiong 已提交
1916
	RT_CACHE_STAT_INC(in_slow_tot);
L
Linus Torvalds 已提交
1917
	if (res.type == RTN_UNREACHABLE) {
1918 1919
		rth->dst.input= ip_error;
		rth->dst.error= -err;
L
Linus Torvalds 已提交
1920 1921
		rth->rt_flags 	&= ~RTCF_LOCAL;
	}
1922 1923 1924 1925 1926 1927
	if (do_cache) {
		if (unlikely(!rt_cache_route(&FIB_RES_NH(res), rth))) {
			rth->dst.flags |= DST_NOCACHE;
			rt_add_uncached_list(rth);
		}
	}
D
David S. Miller 已提交
1928
	skb_dst_set(skb, &rth->dst);
1929
	err = 0;
E
Eric Dumazet 已提交
1930
	goto out;
L
Linus Torvalds 已提交
1931 1932 1933 1934

no_route:
	RT_CACHE_STAT_INC(in_no_route);
	res.type = RTN_UNREACHABLE;
1935
	res.fi = NULL;
1936
	res.table = NULL;
L
Linus Torvalds 已提交
1937 1938 1939 1940 1941 1942 1943 1944
	goto local_input;

	/*
	 *	Do not cache martian addresses: they should be logged (RFC1812)
	 */
martian_destination:
	RT_CACHE_STAT_INC(in_martian_dst);
#ifdef CONFIG_IP_ROUTE_VERBOSE
1945 1946 1947
	if (IN_DEV_LOG_MARTIANS(in_dev))
		net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
				     &daddr, &saddr, dev->name);
L
Linus Torvalds 已提交
1948
#endif
1949

L
Linus Torvalds 已提交
1950 1951
e_inval:
	err = -EINVAL;
E
Eric Dumazet 已提交
1952
	goto out;
L
Linus Torvalds 已提交
1953 1954 1955

e_nobufs:
	err = -ENOBUFS;
E
Eric Dumazet 已提交
1956
	goto out;
L
Linus Torvalds 已提交
1957 1958 1959

martian_source:
	ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
E
Eric Dumazet 已提交
1960
	goto out;
L
Linus Torvalds 已提交
1961 1962
}

1963 1964
int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
			 u8 tos, struct net_device *dev)
L
Linus Torvalds 已提交
1965
{
1966
	int res;
L
Linus Torvalds 已提交
1967

1968 1969
	rcu_read_lock();

L
Linus Torvalds 已提交
1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980
	/* Multicast recognition logic is moved from route cache to here.
	   The problem was that too many Ethernet cards have broken/missing
	   hardware multicast filters :-( As result the host on multicasting
	   network acquires a lot of useless route cache entries, sort of
	   SDR messages from all the world. Now we try to get rid of them.
	   Really, provided software IP multicast filter is organized
	   reasonably (at least, hashed), it does not result in a slowdown
	   comparing with route cache reject entries.
	   Note, that multicast routers are not affected, because
	   route cache entry is created eventually.
	 */
1981
	if (ipv4_is_multicast(daddr)) {
1982
		struct in_device *in_dev = __in_dev_get_rcu(dev);
L
Linus Torvalds 已提交
1983

1984
		if (in_dev) {
1985 1986
			int our = ip_check_mc_rcu(in_dev, daddr, saddr,
						  ip_hdr(skb)->protocol);
L
Linus Torvalds 已提交
1987 1988
			if (our
#ifdef CONFIG_IP_MROUTE
1989 1990 1991
				||
			    (!ipv4_is_local_multicast(daddr) &&
			     IN_DEV_MFORWARD(in_dev))
L
Linus Torvalds 已提交
1992
#endif
1993
			   ) {
1994 1995
				int res = ip_route_input_mc(skb, daddr, saddr,
							    tos, dev, our);
L
Linus Torvalds 已提交
1996
				rcu_read_unlock();
1997
				return res;
L
Linus Torvalds 已提交
1998 1999 2000 2001 2002
			}
		}
		rcu_read_unlock();
		return -EINVAL;
	}
2003
	res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2004 2005
	rcu_read_unlock();
	return res;
L
Linus Torvalds 已提交
2006
}
2007
EXPORT_SYMBOL(ip_route_input_noref);
L
Linus Torvalds 已提交
2008

E
Eric Dumazet 已提交
2009
/* called with rcu_read_lock() */
2010
static struct rtable *__mkroute_output(const struct fib_result *res,
2011
				       const struct flowi4 *fl4, int orig_oif,
2012
				       struct net_device *dev_out,
2013
				       unsigned int flags)
L
Linus Torvalds 已提交
2014
{
2015
	struct fib_info *fi = res->fi;
2016
	struct fib_nh_exception *fnhe;
2017
	struct in_device *in_dev;
2018
	u16 type = res->type;
2019
	struct rtable *rth;
J
Julian Anastasov 已提交
2020
	bool do_cache;
L
Linus Torvalds 已提交
2021

2022 2023
	in_dev = __in_dev_get_rcu(dev_out);
	if (!in_dev)
2024
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
2025

2026
	if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2027 2028 2029
		if (ipv4_is_loopback(fl4->saddr) &&
		    !(dev_out->flags & IFF_LOOPBACK) &&
		    !netif_is_l3_master(dev_out))
2030 2031
			return ERR_PTR(-EINVAL);

2032
	if (ipv4_is_lbcast(fl4->daddr))
2033
		type = RTN_BROADCAST;
2034
	else if (ipv4_is_multicast(fl4->daddr))
2035
		type = RTN_MULTICAST;
2036
	else if (ipv4_is_zeronet(fl4->daddr))
2037
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
2038 2039 2040 2041

	if (dev_out->flags & IFF_LOOPBACK)
		flags |= RTCF_LOCAL;

2042
	do_cache = true;
2043
	if (type == RTN_BROADCAST) {
L
Linus Torvalds 已提交
2044
		flags |= RTCF_BROADCAST | RTCF_LOCAL;
2045 2046
		fi = NULL;
	} else if (type == RTN_MULTICAST) {
E
Eric Dumazet 已提交
2047
		flags |= RTCF_MULTICAST | RTCF_LOCAL;
2048 2049
		if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
				     fl4->flowi4_proto))
L
Linus Torvalds 已提交
2050
			flags &= ~RTCF_LOCAL;
2051 2052
		else
			do_cache = false;
L
Linus Torvalds 已提交
2053
		/* If multicast route do not exist use
E
Eric Dumazet 已提交
2054 2055
		 * default one, but do not gateway in this case.
		 * Yes, it is hack.
L
Linus Torvalds 已提交
2056
		 */
2057 2058
		if (fi && res->prefixlen < 4)
			fi = NULL;
2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070
	} else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
		   (orig_oif != dev_out->ifindex)) {
		/* For local routes that require a particular output interface
		 * we do not want to cache the result.  Caching the result
		 * causes incorrect behaviour when there are multiple source
		 * addresses on the interface, the end result being that if the
		 * intended recipient is waiting on that interface for the
		 * packet he won't receive it because it will be delivered on
		 * the loopback interface and the IP_PKTINFO ipi_ifindex will
		 * be set to the loopback interface as well.
		 */
		fi = NULL;
L
Linus Torvalds 已提交
2071 2072
	}

2073
	fnhe = NULL;
2074 2075
	do_cache &= fi != NULL;
	if (do_cache) {
2076
		struct rtable __rcu **prth;
J
Julian Anastasov 已提交
2077
		struct fib_nh *nh = &FIB_RES_NH(*res);
E
Eric Dumazet 已提交
2078

J
Julian Anastasov 已提交
2079
		fnhe = find_exception(nh, fl4->daddr);
2080
		if (fnhe) {
2081
			prth = &fnhe->fnhe_rth_output;
2082 2083 2084 2085 2086 2087 2088
			rth = rcu_dereference(*prth);
			if (rth && rth->dst.expires &&
			    time_after(jiffies, rth->dst.expires)) {
				ip_del_fnhe(nh, fl4->daddr);
				fnhe = NULL;
			} else {
				goto rt_cache;
J
Julian Anastasov 已提交
2089 2090
			}
		}
2091 2092 2093 2094 2095 2096 2097 2098 2099

		if (unlikely(fl4->flowi4_flags &
			     FLOWI_FLAG_KNOWN_NH &&
			     !(nh->nh_gw &&
			       nh->nh_scope == RT_SCOPE_LINK))) {
			do_cache = false;
			goto add;
		}
		prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2100
		rth = rcu_dereference(*prth);
2101 2102

rt_cache:
2103 2104 2105
		if (rt_cache_valid(rth)) {
			dst_hold(&rth->dst);
			return rth;
2106 2107
		}
	}
J
Julian Anastasov 已提交
2108 2109

add:
2110
	rth = rt_dst_alloc(dev_out, flags, type,
2111
			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
2112
			   IN_DEV_CONF_GET(in_dev, NOXFRM),
J
Julian Anastasov 已提交
2113
			   do_cache);
2114
	if (!rth)
2115
		return ERR_PTR(-ENOBUFS);
2116

2117
	rth->rt_iif	= orig_oif ? : 0;
D
David Ahern 已提交
2118 2119 2120
	if (res->table)
		rth->rt_table_id = res->table->tb_id;

L
Linus Torvalds 已提交
2121 2122 2123
	RT_CACHE_STAT_INC(out_slow_tot);

	if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2124
		if (flags & RTCF_LOCAL &&
L
Linus Torvalds 已提交
2125
		    !(dev_out->flags & IFF_LOOPBACK)) {
2126
			rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2127 2128 2129
			RT_CACHE_STAT_INC(out_slow_mc);
		}
#ifdef CONFIG_IP_MROUTE
2130
		if (type == RTN_MULTICAST) {
L
Linus Torvalds 已提交
2131
			if (IN_DEV_MFORWARD(in_dev) &&
2132
			    !ipv4_is_local_multicast(fl4->daddr)) {
2133 2134
				rth->dst.input = ip_mr_input;
				rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2135 2136 2137 2138 2139
			}
		}
#endif
	}

2140
	rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
2141
	if (lwtunnel_output_redirect(rth->dst.lwtstate))
2142
		rth->dst.output = lwtunnel_output;
L
Linus Torvalds 已提交
2143

2144
	return rth;
L
Linus Torvalds 已提交
2145 2146 2147 2148 2149 2150
}

/*
 * Major route resolver routine.
 */

2151 2152
struct rtable *__ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
					  int mp_hash)
L
Linus Torvalds 已提交
2153 2154
{
	struct net_device *dev_out = NULL;
2155
	__u8 tos = RT_FL_TOS(fl4);
2156 2157
	unsigned int flags = 0;
	struct fib_result res;
2158
	struct rtable *rth;
2159
	int orig_oif;
2160
	int err = -ENETUNREACH;
L
Linus Torvalds 已提交
2161

2162
	res.tclassid	= 0;
L
Linus Torvalds 已提交
2163
	res.fi		= NULL;
2164
	res.table	= NULL;
L
Linus Torvalds 已提交
2165

2166 2167
	orig_oif = fl4->flowi4_oif;

2168
	fl4->flowi4_iif = LOOPBACK_IFINDEX;
2169 2170 2171
	fl4->flowi4_tos = tos & IPTOS_RT_MASK;
	fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
			 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2172

2173
	rcu_read_lock();
2174
	if (fl4->saddr) {
2175
		rth = ERR_PTR(-EINVAL);
2176 2177 2178
		if (ipv4_is_multicast(fl4->saddr) ||
		    ipv4_is_lbcast(fl4->saddr) ||
		    ipv4_is_zeronet(fl4->saddr))
L
Linus Torvalds 已提交
2179 2180 2181 2182
			goto out;

		/* I removed check for oif == dev_out->oif here.
		   It was wrong for two reasons:
2183 2184
		   1. ip_dev_find(net, saddr) can return wrong iface, if saddr
		      is assigned to multiple interfaces.
L
Linus Torvalds 已提交
2185 2186 2187 2188
		   2. Moreover, we are allowed to send packets with saddr
		      of another iface. --ANK
		 */

2189 2190 2191
		if (fl4->flowi4_oif == 0 &&
		    (ipv4_is_multicast(fl4->daddr) ||
		     ipv4_is_lbcast(fl4->daddr))) {
2192
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2193
			dev_out = __ip_dev_find(net, fl4->saddr, false);
2194
			if (!dev_out)
2195 2196
				goto out;

L
Linus Torvalds 已提交
2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
			/* Special hack: user can direct multicasts
			   and limited broadcast via necessary interface
			   without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
			   This hack is not just for fun, it allows
			   vic,vat and friends to work.
			   They bind socket to loopback, set ttl to zero
			   and expect that it will work.
			   From the viewpoint of routing cache they are broken,
			   because we are not allowed to build multicast path
			   with loopback source addr (look, routing cache
			   cannot know, that ttl is zero, so that packet
			   will not leave this host and route is valid).
			   Luckily, this hack is good workaround.
			 */

2212
			fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2213 2214
			goto make_route;
		}
2215

2216
		if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2217
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2218
			if (!__ip_dev_find(net, fl4->saddr, false))
2219 2220
				goto out;
		}
L
Linus Torvalds 已提交
2221 2222 2223
	}


2224 2225
	if (fl4->flowi4_oif) {
		dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2226
		rth = ERR_PTR(-ENODEV);
2227
		if (!dev_out)
L
Linus Torvalds 已提交
2228
			goto out;
2229 2230

		/* RACE: Check return value of inet_select_addr instead. */
2231
		if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2232
			rth = ERR_PTR(-ENETUNREACH);
2233 2234
			goto out;
		}
2235
		if (ipv4_is_local_multicast(fl4->daddr) ||
2236 2237
		    ipv4_is_lbcast(fl4->daddr) ||
		    fl4->flowi4_proto == IPPROTO_IGMP) {
2238 2239 2240
			if (!fl4->saddr)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
L
Linus Torvalds 已提交
2241 2242
			goto make_route;
		}
2243
		if (!fl4->saddr) {
2244 2245 2246 2247 2248 2249
			if (ipv4_is_multicast(fl4->daddr))
				fl4->saddr = inet_select_addr(dev_out, 0,
							      fl4->flowi4_scope);
			else if (!fl4->daddr)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_HOST);
L
Linus Torvalds 已提交
2250 2251 2252
		}
	}

2253 2254 2255 2256
	if (!fl4->daddr) {
		fl4->daddr = fl4->saddr;
		if (!fl4->daddr)
			fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2257
		dev_out = net->loopback_dev;
2258
		fl4->flowi4_oif = LOOPBACK_IFINDEX;
L
Linus Torvalds 已提交
2259 2260 2261 2262 2263
		res.type = RTN_LOCAL;
		flags |= RTCF_LOCAL;
		goto make_route;
	}

2264 2265
	err = fib_lookup(net, fl4, &res, 0);
	if (err) {
L
Linus Torvalds 已提交
2266
		res.fi = NULL;
2267
		res.table = NULL;
2268
		if (fl4->flowi4_oif) {
L
Linus Torvalds 已提交
2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
			/* Apparently, routing tables are wrong. Assume,
			   that the destination is on link.

			   WHY? DW.
			   Because we are allowed to send to iface
			   even if it has NO routes and NO assigned
			   addresses. When oif is specified, routing
			   tables are looked up with only one purpose:
			   to catch if destination is gatewayed, rather than
			   direct. Moreover, if MSG_DONTROUTE is set,
			   we send packet, ignoring both routing tables
			   and ifaddr state. --ANK


			   We could make it even if oif is unknown,
			   likely IPv6, but we do not.
			 */

2287 2288 2289
			if (fl4->saddr == 0)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
L
Linus Torvalds 已提交
2290 2291 2292
			res.type = RTN_UNICAST;
			goto make_route;
		}
2293
		rth = ERR_PTR(err);
L
Linus Torvalds 已提交
2294 2295 2296 2297
		goto out;
	}

	if (res.type == RTN_LOCAL) {
2298
		if (!fl4->saddr) {
2299
			if (res.fi->fib_prefsrc)
2300
				fl4->saddr = res.fi->fib_prefsrc;
2301
			else
2302
				fl4->saddr = fl4->daddr;
2303
		}
2304 2305 2306

		/* L3 master device is the loopback for that domain */
		dev_out = l3mdev_master_dev_rcu(dev_out) ? : net->loopback_dev;
2307
		fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2308 2309 2310 2311
		flags |= RTCF_LOCAL;
		goto make_route;
	}

2312
	fib_select_path(net, &res, fl4, mp_hash);
L
Linus Torvalds 已提交
2313 2314

	dev_out = FIB_RES_DEV(res);
2315
	fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2316 2317 2318


make_route:
2319
	rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
L
Linus Torvalds 已提交
2320

2321 2322
out:
	rcu_read_unlock();
2323
	return rth;
L
Linus Torvalds 已提交
2324
}
2325
EXPORT_SYMBOL_GPL(__ip_route_output_key_hash);
2326

2327 2328 2329 2330 2331
static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
{
	return NULL;
}

2332
static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2333
{
2334 2335 2336
	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);

	return mtu ? : dst->dev->mtu;
2337 2338
}

2339 2340
static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
					  struct sk_buff *skb, u32 mtu)
2341 2342 2343
{
}

2344 2345
static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
				       struct sk_buff *skb)
2346 2347 2348
{
}

2349 2350 2351 2352 2353 2354
static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
					  unsigned long old)
{
	return NULL;
}

2355 2356
static struct dst_ops ipv4_dst_blackhole_ops = {
	.family			=	AF_INET,
2357
	.check			=	ipv4_blackhole_dst_check,
2358
	.mtu			=	ipv4_blackhole_mtu,
2359
	.default_advmss		=	ipv4_default_advmss,
2360
	.update_pmtu		=	ipv4_rt_blackhole_update_pmtu,
2361
	.redirect		=	ipv4_rt_blackhole_redirect,
2362
	.cow_metrics		=	ipv4_rt_blackhole_cow_metrics,
2363
	.neigh_lookup		=	ipv4_neigh_lookup,
2364 2365
};

2366
struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2367
{
2368
	struct rtable *ort = (struct rtable *) dst_orig;
2369
	struct rtable *rt;
2370

2371
	rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2372
	if (rt) {
2373
		struct dst_entry *new = &rt->dst;
2374 2375

		new->__use = 1;
2376
		new->input = dst_discard;
E
Eric W. Biederman 已提交
2377
		new->output = dst_discard_out;
2378

2379
		new->dev = ort->dst.dev;
2380 2381 2382
		if (new->dev)
			dev_hold(new->dev);

2383
		rt->rt_is_input = ort->rt_is_input;
2384
		rt->rt_iif = ort->rt_iif;
2385
		rt->rt_pmtu = ort->rt_pmtu;
2386

F
fan.du 已提交
2387
		rt->rt_genid = rt_genid_ipv4(net);
2388 2389 2390
		rt->rt_flags = ort->rt_flags;
		rt->rt_type = ort->rt_type;
		rt->rt_gateway = ort->rt_gateway;
J
Julian Anastasov 已提交
2391
		rt->rt_uses_gateway = ort->rt_uses_gateway;
2392

2393
		INIT_LIST_HEAD(&rt->rt_uncached);
2394 2395 2396
		dst_free(new);
	}

2397 2398 2399
	dst_release(dst_orig);

	return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2400 2401
}

2402
struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2403
				    const struct sock *sk)
L
Linus Torvalds 已提交
2404
{
2405
	struct rtable *rt = __ip_route_output_key(net, flp4);
L
Linus Torvalds 已提交
2406

2407 2408
	if (IS_ERR(rt))
		return rt;
L
Linus Torvalds 已提交
2409

2410
	if (flp4->flowi4_proto)
2411 2412 2413
		rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
							flowi4_to_flowi(flp4),
							sk, 0);
L
Linus Torvalds 已提交
2414

2415
	return rt;
L
Linus Torvalds 已提交
2416
}
2417 2418
EXPORT_SYMBOL_GPL(ip_route_output_flow);

2419
static int rt_fill_info(struct net *net,  __be32 dst, __be32 src, u32 table_id,
2420
			struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2421
			u32 seq, int event, int nowait, unsigned int flags)
L
Linus Torvalds 已提交
2422
{
E
Eric Dumazet 已提交
2423
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
2424
	struct rtmsg *r;
2425
	struct nlmsghdr *nlh;
2426
	unsigned long expires = 0;
2427
	u32 error;
J
Julian Anastasov 已提交
2428
	u32 metrics[RTAX_MAX];
2429

2430
	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
2431
	if (!nlh)
2432
		return -EMSGSIZE;
2433 2434

	r = nlmsg_data(nlh);
L
Linus Torvalds 已提交
2435 2436 2437
	r->rtm_family	 = AF_INET;
	r->rtm_dst_len	= 32;
	r->rtm_src_len	= 0;
2438
	r->rtm_tos	= fl4->flowi4_tos;
2439 2440
	r->rtm_table	= table_id;
	if (nla_put_u32(skb, RTA_TABLE, table_id))
D
David S. Miller 已提交
2441
		goto nla_put_failure;
L
Linus Torvalds 已提交
2442 2443 2444 2445 2446 2447
	r->rtm_type	= rt->rt_type;
	r->rtm_scope	= RT_SCOPE_UNIVERSE;
	r->rtm_protocol = RTPROT_UNSPEC;
	r->rtm_flags	= (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
	if (rt->rt_flags & RTCF_NOTIFY)
		r->rtm_flags |= RTM_F_NOTIFY;
2448 2449
	if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
		r->rtm_flags |= RTCF_DOREDIRECT;
2450

2451
	if (nla_put_in_addr(skb, RTA_DST, dst))
D
David S. Miller 已提交
2452
		goto nla_put_failure;
2453
	if (src) {
L
Linus Torvalds 已提交
2454
		r->rtm_src_len = 32;
2455
		if (nla_put_in_addr(skb, RTA_SRC, src))
D
David S. Miller 已提交
2456
			goto nla_put_failure;
L
Linus Torvalds 已提交
2457
	}
D
David S. Miller 已提交
2458 2459 2460
	if (rt->dst.dev &&
	    nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
		goto nla_put_failure;
2461
#ifdef CONFIG_IP_ROUTE_CLASSID
D
David S. Miller 已提交
2462 2463 2464
	if (rt->dst.tclassid &&
	    nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
		goto nla_put_failure;
L
Linus Torvalds 已提交
2465
#endif
2466
	if (!rt_is_input_route(rt) &&
2467
	    fl4->saddr != src) {
2468
		if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
D
David S. Miller 已提交
2469 2470
			goto nla_put_failure;
	}
J
Julian Anastasov 已提交
2471
	if (rt->rt_uses_gateway &&
2472
	    nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
D
David S. Miller 已提交
2473
		goto nla_put_failure;
2474

2475 2476 2477 2478 2479 2480 2481 2482 2483 2484
	expires = rt->dst.expires;
	if (expires) {
		unsigned long now = jiffies;

		if (time_before(now, expires))
			expires -= now;
		else
			expires = 0;
	}

J
Julian Anastasov 已提交
2485
	memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2486
	if (rt->rt_pmtu && expires)
J
Julian Anastasov 已提交
2487 2488
		metrics[RTAX_MTU - 1] = rt->rt_pmtu;
	if (rtnetlink_put_metrics(skb, metrics) < 0)
2489 2490
		goto nla_put_failure;

2491
	if (fl4->flowi4_mark &&
2492
	    nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
D
David S. Miller 已提交
2493
		goto nla_put_failure;
E
Eric Dumazet 已提交
2494

2495
	error = rt->dst.error;
2496

2497
	if (rt_is_input_route(rt)) {
2498 2499 2500 2501 2502
#ifdef CONFIG_IP_MROUTE
		if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
		    IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
			int err = ipmr_get_route(net, skb,
						 fl4->saddr, fl4->daddr,
2503 2504
						 r, nowait, portid);

2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517
			if (err <= 0) {
				if (!nowait) {
					if (err == 0)
						return 0;
					goto nla_put_failure;
				} else {
					if (err == -EMSGSIZE)
						goto nla_put_failure;
					error = err;
				}
			}
		} else
#endif
2518
			if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2519
				goto nla_put_failure;
L
Linus Torvalds 已提交
2520 2521
	}

2522
	if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2523
		goto nla_put_failure;
2524

2525 2526
	nlmsg_end(skb, nlh);
	return 0;
L
Linus Torvalds 已提交
2527

2528
nla_put_failure:
2529 2530
	nlmsg_cancel(skb, nlh);
	return -EMSGSIZE;
L
Linus Torvalds 已提交
2531 2532
}

2533
static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
L
Linus Torvalds 已提交
2534
{
2535
	struct net *net = sock_net(in_skb->sk);
2536 2537
	struct rtmsg *rtm;
	struct nlattr *tb[RTA_MAX+1];
L
Linus Torvalds 已提交
2538
	struct rtable *rt = NULL;
2539
	struct flowi4 fl4;
A
Al Viro 已提交
2540 2541 2542
	__be32 dst = 0;
	__be32 src = 0;
	u32 iif;
2543
	int err;
E
Eric Dumazet 已提交
2544
	int mark;
L
Linus Torvalds 已提交
2545
	struct sk_buff *skb;
2546
	u32 table_id = RT_TABLE_MAIN;
L
Linus Torvalds 已提交
2547

2548 2549 2550 2551 2552 2553
	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
	if (err < 0)
		goto errout;

	rtm = nlmsg_data(nlh);

L
Linus Torvalds 已提交
2554
	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2555
	if (!skb) {
2556 2557 2558
		err = -ENOBUFS;
		goto errout;
	}
L
Linus Torvalds 已提交
2559 2560 2561 2562

	/* Reserve room for dummy headers, this skb can pass
	   through good chunk of routing engine.
	 */
2563
	skb_reset_mac_header(skb);
2564
	skb_reset_network_header(skb);
2565 2566

	/* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2567
	ip_hdr(skb)->protocol = IPPROTO_ICMP;
L
Linus Torvalds 已提交
2568 2569
	skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));

2570 2571
	src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
	dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2572
	iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
E
Eric Dumazet 已提交
2573
	mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
L
Linus Torvalds 已提交
2574

2575 2576 2577 2578 2579 2580 2581
	memset(&fl4, 0, sizeof(fl4));
	fl4.daddr = dst;
	fl4.saddr = src;
	fl4.flowi4_tos = rtm->rtm_tos;
	fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
	fl4.flowi4_mark = mark;

L
Linus Torvalds 已提交
2582
	if (iif) {
2583 2584
		struct net_device *dev;

2585
		dev = __dev_get_by_index(net, iif);
2586
		if (!dev) {
2587 2588 2589 2590
			err = -ENODEV;
			goto errout_free;
		}

L
Linus Torvalds 已提交
2591 2592
		skb->protocol	= htons(ETH_P_IP);
		skb->dev	= dev;
E
Eric Dumazet 已提交
2593
		skb->mark	= mark;
L
Linus Torvalds 已提交
2594 2595 2596
		local_bh_disable();
		err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
		local_bh_enable();
2597

E
Eric Dumazet 已提交
2598
		rt = skb_rtable(skb);
2599 2600
		if (err == 0 && rt->dst.error)
			err = -rt->dst.error;
L
Linus Torvalds 已提交
2601
	} else {
2602
		rt = ip_route_output_key(net, &fl4);
2603 2604 2605 2606

		err = 0;
		if (IS_ERR(rt))
			err = PTR_ERR(rt);
L
Linus Torvalds 已提交
2607
	}
2608

L
Linus Torvalds 已提交
2609
	if (err)
2610
		goto errout_free;
L
Linus Torvalds 已提交
2611

2612
	skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
2613 2614 2615
	if (rtm->rtm_flags & RTM_F_NOTIFY)
		rt->rt_flags |= RTCF_NOTIFY;

2616 2617 2618 2619
	if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
		table_id = rt->rt_table_id;

	err = rt_fill_info(net, dst, src, table_id, &fl4, skb,
2620
			   NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2621
			   RTM_NEWROUTE, 0, 0);
2622
	if (err < 0)
2623
		goto errout_free;
L
Linus Torvalds 已提交
2624

2625
	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2626
errout:
2627
	return err;
L
Linus Torvalds 已提交
2628

2629
errout_free:
L
Linus Torvalds 已提交
2630
	kfree_skb(skb);
2631
	goto errout;
L
Linus Torvalds 已提交
2632 2633 2634 2635
}

void ip_rt_multicast_event(struct in_device *in_dev)
{
2636
	rt_cache_flush(dev_net(in_dev->dev));
L
Linus Torvalds 已提交
2637 2638 2639
}

#ifdef CONFIG_SYSCTL
2640 2641 2642 2643
static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
static int ip_rt_gc_min_interval __read_mostly	= HZ / 2;
static int ip_rt_gc_elasticity __read_mostly	= 8;

2644
static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2645
					void __user *buffer,
L
Linus Torvalds 已提交
2646 2647
					size_t *lenp, loff_t *ppos)
{
2648 2649
	struct net *net = (struct net *)__ctl->extra1;

L
Linus Torvalds 已提交
2650
	if (write) {
2651 2652
		rt_cache_flush(net);
		fnhe_genid_bump(net);
L
Linus Torvalds 已提交
2653
		return 0;
2654
	}
L
Linus Torvalds 已提交
2655 2656 2657 2658

	return -EINVAL;
}

2659
static struct ctl_table ipv4_route_table[] = {
L
Linus Torvalds 已提交
2660 2661 2662 2663 2664
	{
		.procname	= "gc_thresh",
		.data		= &ipv4_dst_ops.gc_thresh,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2665
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2666 2667 2668 2669 2670 2671
	},
	{
		.procname	= "max_size",
		.data		= &ip_rt_max_size,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2672
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2673 2674 2675
	},
	{
		/*  Deprecated. Use gc_min_interval_ms */
2676

L
Linus Torvalds 已提交
2677 2678 2679 2680
		.procname	= "gc_min_interval",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2681
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
2682 2683 2684 2685 2686 2687
	},
	{
		.procname	= "gc_min_interval_ms",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2688
		.proc_handler	= proc_dointvec_ms_jiffies,
L
Linus Torvalds 已提交
2689 2690 2691 2692 2693 2694
	},
	{
		.procname	= "gc_timeout",
		.data		= &ip_rt_gc_timeout,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2695
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
2696
	},
2697 2698 2699 2700 2701 2702 2703
	{
		.procname	= "gc_interval",
		.data		= &ip_rt_gc_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_jiffies,
	},
L
Linus Torvalds 已提交
2704 2705 2706 2707 2708
	{
		.procname	= "redirect_load",
		.data		= &ip_rt_redirect_load,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2709
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2710 2711 2712 2713 2714 2715
	},
	{
		.procname	= "redirect_number",
		.data		= &ip_rt_redirect_number,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2716
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2717 2718 2719 2720 2721 2722
	},
	{
		.procname	= "redirect_silence",
		.data		= &ip_rt_redirect_silence,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2723
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2724 2725 2726 2727 2728 2729
	},
	{
		.procname	= "error_cost",
		.data		= &ip_rt_error_cost,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2730
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2731 2732 2733 2734 2735 2736
	},
	{
		.procname	= "error_burst",
		.data		= &ip_rt_error_burst,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2737
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2738 2739 2740 2741 2742 2743
	},
	{
		.procname	= "gc_elasticity",
		.data		= &ip_rt_gc_elasticity,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2744
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2745 2746 2747 2748 2749 2750
	},
	{
		.procname	= "mtu_expires",
		.data		= &ip_rt_mtu_expires,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2751
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
2752 2753 2754 2755 2756 2757
	},
	{
		.procname	= "min_pmtu",
		.data		= &ip_rt_min_pmtu,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2758
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2759 2760 2761 2762 2763 2764
	},
	{
		.procname	= "min_adv_mss",
		.data		= &ip_rt_min_advmss,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2765
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2766
	},
2767
	{ }
L
Linus Torvalds 已提交
2768
};
2769 2770 2771 2772 2773 2774

static struct ctl_table ipv4_route_flush_table[] = {
	{
		.procname	= "flush",
		.maxlen		= sizeof(int),
		.mode		= 0200,
A
Alexey Dobriyan 已提交
2775
		.proc_handler	= ipv4_sysctl_rtcache_flush,
2776
	},
2777
	{ },
2778 2779 2780 2781 2782 2783 2784
};

static __net_init int sysctl_route_net_init(struct net *net)
{
	struct ctl_table *tbl;

	tbl = ipv4_route_flush_table;
O
Octavian Purdila 已提交
2785
	if (!net_eq(net, &init_net)) {
2786
		tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2787
		if (!tbl)
2788
			goto err_dup;
2789 2790 2791 2792

		/* Don't export sysctls to unprivileged users */
		if (net->user_ns != &init_user_ns)
			tbl[0].procname = NULL;
2793 2794 2795
	}
	tbl[0].extra1 = net;

2796
	net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2797
	if (!net->ipv4.route_hdr)
2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821
		goto err_reg;
	return 0;

err_reg:
	if (tbl != ipv4_route_flush_table)
		kfree(tbl);
err_dup:
	return -ENOMEM;
}

static __net_exit void sysctl_route_net_exit(struct net *net)
{
	struct ctl_table *tbl;

	tbl = net->ipv4.route_hdr->ctl_table_arg;
	unregister_net_sysctl_table(net->ipv4.route_hdr);
	BUG_ON(tbl == ipv4_route_flush_table);
	kfree(tbl);
}

static __net_initdata struct pernet_operations sysctl_route_ops = {
	.init = sysctl_route_net_init,
	.exit = sysctl_route_net_exit,
};
L
Linus Torvalds 已提交
2822 2823
#endif

2824
static __net_init int rt_genid_init(struct net *net)
2825
{
F
fan.du 已提交
2826
	atomic_set(&net->ipv4.rt_genid, 0);
2827
	atomic_set(&net->fnhe_genid, 0);
2828 2829
	get_random_bytes(&net->ipv4.dev_addr_genid,
			 sizeof(net->ipv4.dev_addr_genid));
2830 2831 2832
	return 0;
}

2833 2834
static __net_initdata struct pernet_operations rt_genid_ops = {
	.init = rt_genid_init,
2835 2836
};

2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852
static int __net_init ipv4_inetpeer_init(struct net *net)
{
	struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);

	if (!bp)
		return -ENOMEM;
	inet_peer_base_init(bp);
	net->ipv4.peers = bp;
	return 0;
}

static void __net_exit ipv4_inetpeer_exit(struct net *net)
{
	struct inet_peer_base *bp = net->ipv4.peers;

	net->ipv4.peers = NULL;
2853
	inetpeer_invalidate_tree(bp);
2854 2855 2856 2857 2858 2859 2860
	kfree(bp);
}

static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
	.init	=	ipv4_inetpeer_init,
	.exit	=	ipv4_inetpeer_exit,
};
2861

2862
#ifdef CONFIG_IP_ROUTE_CLASSID
2863
struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2864
#endif /* CONFIG_IP_ROUTE_CLASSID */
L
Linus Torvalds 已提交
2865 2866 2867

int __init ip_rt_init(void)
{
2868
	int rc = 0;
E
Eric Dumazet 已提交
2869
	int cpu;
L
Linus Torvalds 已提交
2870

E
Eric Dumazet 已提交
2871 2872 2873 2874 2875 2876
	ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
	if (!ip_idents)
		panic("IP: failed to allocate ip_idents\n");

	prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));

E
Eric Dumazet 已提交
2877 2878 2879 2880
	ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
	if (!ip_tstamps)
		panic("IP: failed to allocate ip_tstamps\n");

E
Eric Dumazet 已提交
2881 2882 2883 2884 2885 2886
	for_each_possible_cpu(cpu) {
		struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);

		INIT_LIST_HEAD(&ul->head);
		spin_lock_init(&ul->lock);
	}
2887
#ifdef CONFIG_IP_ROUTE_CLASSID
2888
	ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
L
Linus Torvalds 已提交
2889 2890 2891 2892
	if (!ip_rt_acct)
		panic("IP: failed to allocate ip_rt_acct\n");
#endif

A
Alexey Dobriyan 已提交
2893 2894
	ipv4_dst_ops.kmem_cachep =
		kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2895
				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
2896

2897 2898
	ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;

2899 2900 2901 2902 2903 2904
	if (dst_entries_init(&ipv4_dst_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_ops counter\n");

	if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");

D
David S. Miller 已提交
2905 2906
	ipv4_dst_ops.gc_thresh = ~0;
	ip_rt_max_size = INT_MAX;
L
Linus Torvalds 已提交
2907 2908 2909 2910

	devinet_init();
	ip_fib_init();

2911
	if (ip_rt_proc_init())
J
Joe Perches 已提交
2912
		pr_err("Unable to create route proc files\n");
L
Linus Torvalds 已提交
2913 2914
#ifdef CONFIG_XFRM
	xfrm_init();
2915
	xfrm4_init();
L
Linus Torvalds 已提交
2916
#endif
2917
	rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2918

2919 2920 2921
#ifdef CONFIG_SYSCTL
	register_pernet_subsys(&sysctl_route_ops);
#endif
2922
	register_pernet_subsys(&rt_genid_ops);
2923
	register_pernet_subsys(&ipv4_inetpeer_ops);
L
Linus Torvalds 已提交
2924 2925 2926
	return rc;
}

2927
#ifdef CONFIG_SYSCTL
A
Al Viro 已提交
2928 2929 2930 2931 2932 2933
/*
 * We really need to sanitize the damn ipv4 init order, then all
 * this nonsense will go away.
 */
void __init ip_static_sysctl_init(void)
{
2934
	register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
A
Al Viro 已提交
2935
}
2936
#endif