route.c 74.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		ROUTE - implementation of the IP router.
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Linus Torvalds, <Linus.Torvalds@helsinki.fi>
 *		Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 * Fixes:
 *		Alan Cox	:	Verify area fixes.
 *		Alan Cox	:	cli() protects routing changes
 *		Rui Oliveira	:	ICMP routing table updates
 *		(rco@di.uminho.pt)	Routing table insertion and update
 *		Linus Torvalds	:	Rewrote bits to be sensible
 *		Alan Cox	:	Added BSD route gw semantics
21
 *		Alan Cox	:	Super /proc >4K
L
Linus Torvalds 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
 *		Alan Cox	:	MTU in route table
 *		Alan Cox	: 	MSS actually. Also added the window
 *					clamper.
 *		Sam Lantinga	:	Fixed route matching in rt_del()
 *		Alan Cox	:	Routing cache support.
 *		Alan Cox	:	Removed compatibility cruft.
 *		Alan Cox	:	RTF_REJECT support.
 *		Alan Cox	:	TCP irtt support.
 *		Jonathan Naylor	:	Added Metric support.
 *	Miquel van Smoorenburg	:	BSD API fixes.
 *	Miquel van Smoorenburg	:	Metrics.
 *		Alan Cox	:	Use __u32 properly
 *		Alan Cox	:	Aligned routing errors more closely with BSD
 *					our system is still very different.
 *		Alan Cox	:	Faster /proc handling
 *	Alexey Kuznetsov	:	Massive rework to support tree based routing,
 *					routing caches and better behaviour.
39
 *
L
Linus Torvalds 已提交
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
 *		Olaf Erb	:	irtt wasn't being copied right.
 *		Bjorn Ekwall	:	Kerneld route support.
 *		Alan Cox	:	Multicast fixed (I hope)
 * 		Pavel Krauz	:	Limited broadcast fixed
 *		Mike McLagan	:	Routing by source
 *	Alexey Kuznetsov	:	End of old history. Split to fib.c and
 *					route.c and rewritten from scratch.
 *		Andi Kleen	:	Load-limit warning messages.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
 *	Vitaly E. Lavrov	:	Race condition in ip_route_input_slow.
 *	Tobias Ringstrom	:	Uninitialized res.type in ip_route_output_slow.
 *	Vladimir V. Ivanov	:	IP rule info (flowid) is really useful.
 *		Marc Boucher	:	routing by fwmark
 *	Robert Olsson		:	Added rt_cache statistics
 *	Arnaldo C. Melo		:	Convert proc stuff to seq_file
55
 *	Eric Dumazet		:	hashed spinlocks and rt_check_expire() fixes.
56 57
 * 	Ilia Sotnikov		:	Ignore TOS on PMTUD and Redirect
 * 	Ilia Sotnikov		:	Removed TOS from hash calculations
L
Linus Torvalds 已提交
58 59 60 61 62 63 64
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 */

65 66
#define pr_fmt(fmt) "IPv4: " fmt

L
Linus Torvalds 已提交
67
#include <linux/module.h>
68
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include <linux/pkt_sched.h>
#include <linux/mroute.h>
#include <linux/netfilter_ipv4.h>
#include <linux/random.h>
#include <linux/rcupdate.h>
#include <linux/times.h>
91
#include <linux/slab.h>
E
Eric Dumazet 已提交
92
#include <linux/jhash.h>
93
#include <net/dst.h>
94
#include <net/dst_metadata.h>
95
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104 105
#include <net/protocol.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/inetpeer.h>
#include <net/sock.h>
#include <net/ip_fib.h>
#include <net/arp.h>
#include <net/tcp.h>
#include <net/icmp.h>
#include <net/xfrm.h>
106
#include <net/lwtunnel.h>
107
#include <net/netevent.h>
108
#include <net/rtnetlink.h>
L
Linus Torvalds 已提交
109 110
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
111
#include <linux/kmemleak.h>
L
Linus Torvalds 已提交
112
#endif
113
#include <net/secure_seq.h>
114
#include <net/ip_tunnels.h>
115
#include <net/l3mdev.h>
L
Linus Torvalds 已提交
116

117
#define RT_FL_TOS(oldflp4) \
118
	((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
L
Linus Torvalds 已提交
119 120 121 122

#define RT_GC_TIMEOUT (300*HZ)

static int ip_rt_max_size;
123 124 125 126 127 128 129 130
static int ip_rt_redirect_number __read_mostly	= 9;
static int ip_rt_redirect_load __read_mostly	= HZ / 50;
static int ip_rt_redirect_silence __read_mostly	= ((HZ / 50) << (9 + 1));
static int ip_rt_error_cost __read_mostly	= HZ;
static int ip_rt_error_burst __read_mostly	= 5 * HZ;
static int ip_rt_mtu_expires __read_mostly	= 10 * 60 * HZ;
static int ip_rt_min_pmtu __read_mostly		= 512 + 20 + 20;
static int ip_rt_min_advmss __read_mostly	= 256;
131

132
static int ip_rt_gc_timeout __read_mostly	= RT_GC_TIMEOUT;
L
Linus Torvalds 已提交
133 134 135 136 137
/*
 *	Interface to generic destination cache.
 */

static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
138
static unsigned int	 ipv4_default_advmss(const struct dst_entry *dst);
139
static unsigned int	 ipv4_mtu(const struct dst_entry *dst);
L
Linus Torvalds 已提交
140 141
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
static void		 ipv4_link_failure(struct sk_buff *skb);
142 143 144 145
static void		 ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
					   struct sk_buff *skb, u32 mtu);
static void		 ip_do_redirect(struct dst_entry *dst, struct sock *sk,
					struct sk_buff *skb);
146
static void		ipv4_dst_destroy(struct dst_entry *dst);
L
Linus Torvalds 已提交
147

148 149
static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
{
150 151
	WARN_ON(1);
	return NULL;
152 153
}

154 155 156
static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
					   struct sk_buff *skb,
					   const void *daddr);
157
static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
158

L
Linus Torvalds 已提交
159 160 161
static struct dst_ops ipv4_dst_ops = {
	.family =		AF_INET,
	.check =		ipv4_dst_check,
162
	.default_advmss =	ipv4_default_advmss,
163
	.mtu =			ipv4_mtu,
164
	.cow_metrics =		ipv4_cow_metrics,
165
	.destroy =		ipv4_dst_destroy,
L
Linus Torvalds 已提交
166 167 168
	.negative_advice =	ipv4_negative_advice,
	.link_failure =		ipv4_link_failure,
	.update_pmtu =		ip_rt_update_pmtu,
169
	.redirect =		ip_do_redirect,
170
	.local_out =		__ip_local_out,
171
	.neigh_lookup =		ipv4_neigh_lookup,
172
	.confirm_neigh =	ipv4_confirm_neigh,
L
Linus Torvalds 已提交
173 174 175 176
};

#define ECN_OR_COST(class)	TC_PRIO_##class

177
const __u8 ip_tos2prio[16] = {
L
Linus Torvalds 已提交
178
	TC_PRIO_BESTEFFORT,
D
Dan Siemon 已提交
179
	ECN_OR_COST(BESTEFFORT),
L
Linus Torvalds 已提交
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	TC_PRIO_BESTEFFORT,
	ECN_OR_COST(BESTEFFORT),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK)
};
195
EXPORT_SYMBOL(ip_tos2prio);
L
Linus Torvalds 已提交
196

197
static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
198
#define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
L
Linus Torvalds 已提交
199 200 201 202

#ifdef CONFIG_PROC_FS
static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
{
203
	if (*pos)
D
David S. Miller 已提交
204
		return NULL;
205
	return SEQ_START_TOKEN;
L
Linus Torvalds 已提交
206 207 208 209 210
}

static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	++*pos;
D
David S. Miller 已提交
211
	return NULL;
L
Linus Torvalds 已提交
212 213 214 215 216 217 218 219 220 221 222 223 224
}

static void rt_cache_seq_stop(struct seq_file *seq, void *v)
{
}

static int rt_cache_seq_show(struct seq_file *seq, void *v)
{
	if (v == SEQ_START_TOKEN)
		seq_printf(seq, "%-127s\n",
			   "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
			   "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
			   "HHUptod\tSpecDst");
225
	return 0;
L
Linus Torvalds 已提交
226 227
}

228
static const struct seq_operations rt_cache_seq_ops = {
L
Linus Torvalds 已提交
229 230 231 232 233 234 235 236
	.start  = rt_cache_seq_start,
	.next   = rt_cache_seq_next,
	.stop   = rt_cache_seq_stop,
	.show   = rt_cache_seq_show,
};

static int rt_cache_seq_open(struct inode *inode, struct file *file)
{
D
David S. Miller 已提交
237
	return seq_open(file, &rt_cache_seq_ops);
L
Linus Torvalds 已提交
238 239
}

240
static const struct file_operations rt_cache_seq_fops = {
L
Linus Torvalds 已提交
241 242 243 244
	.owner	 = THIS_MODULE,
	.open	 = rt_cache_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
D
David S. Miller 已提交
245
	.release = seq_release,
L
Linus Torvalds 已提交
246 247 248 249 250 251 252 253 254 255
};


static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
{
	int cpu;

	if (*pos == 0)
		return SEQ_START_TOKEN;

256
	for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
257 258 259
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
260
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
261 262 263 264 265 266 267 268
	}
	return NULL;
}

static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	int cpu;

269
	for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
270 271 272
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
273
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
274 275
	}
	return NULL;
276

L
Linus Torvalds 已提交
277 278 279 280 281 282 283 284 285 286 287 288
}

static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
{

}

static int rt_cpu_seq_show(struct seq_file *seq, void *v)
{
	struct rt_cache_stat *st = v;

	if (v == SEQ_START_TOKEN) {
289
		seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
L
Linus Torvalds 已提交
290 291
		return 0;
	}
292

L
Linus Torvalds 已提交
293 294
	seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
		   " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
295
		   dst_entries_get_slow(&ipv4_dst_ops),
E
Eric Dumazet 已提交
296
		   0, /* st->in_hit */
L
Linus Torvalds 已提交
297 298 299 300 301 302 303
		   st->in_slow_tot,
		   st->in_slow_mc,
		   st->in_no_route,
		   st->in_brd,
		   st->in_martian_dst,
		   st->in_martian_src,

E
Eric Dumazet 已提交
304
		   0, /* st->out_hit */
L
Linus Torvalds 已提交
305
		   st->out_slow_tot,
306
		   st->out_slow_mc,
L
Linus Torvalds 已提交
307

E
Eric Dumazet 已提交
308 309 310 311 312 313
		   0, /* st->gc_total */
		   0, /* st->gc_ignored */
		   0, /* st->gc_goal_miss */
		   0, /* st->gc_dst_overflow */
		   0, /* st->in_hlist_search */
		   0  /* st->out_hlist_search */
L
Linus Torvalds 已提交
314 315 316 317
		);
	return 0;
}

318
static const struct seq_operations rt_cpu_seq_ops = {
L
Linus Torvalds 已提交
319 320 321 322 323 324 325 326 327 328 329 330
	.start  = rt_cpu_seq_start,
	.next   = rt_cpu_seq_next,
	.stop   = rt_cpu_seq_stop,
	.show   = rt_cpu_seq_show,
};


static int rt_cpu_seq_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &rt_cpu_seq_ops);
}

331
static const struct file_operations rt_cpu_seq_fops = {
L
Linus Torvalds 已提交
332 333 334 335 336 337 338
	.owner	 = THIS_MODULE,
	.open	 = rt_cpu_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
	.release = seq_release,
};

339
#ifdef CONFIG_IP_ROUTE_CLASSID
340
static int rt_acct_proc_show(struct seq_file *m, void *v)
341
{
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
	struct ip_rt_acct *dst, *src;
	unsigned int i, j;

	dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
	if (!dst)
		return -ENOMEM;

	for_each_possible_cpu(i) {
		src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
		for (j = 0; j < 256; j++) {
			dst[j].o_bytes   += src[j].o_bytes;
			dst[j].o_packets += src[j].o_packets;
			dst[j].i_bytes   += src[j].i_bytes;
			dst[j].i_packets += src[j].i_packets;
		}
357 358
	}

359 360 361 362
	seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
	kfree(dst);
	return 0;
}
363

364 365 366
static int rt_acct_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, rt_acct_proc_show, NULL);
367
}
368 369 370 371 372 373 374 375

static const struct file_operations rt_acct_proc_fops = {
	.owner		= THIS_MODULE,
	.open		= rt_acct_proc_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};
376
#endif
377

378
static int __net_init ip_rt_do_proc_init(struct net *net)
379 380 381
{
	struct proc_dir_entry *pde;

382 383
	pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
			  &rt_cache_seq_fops);
384 385 386
	if (!pde)
		goto err1;

387 388
	pde = proc_create("rt_cache", S_IRUGO,
			  net->proc_net_stat, &rt_cpu_seq_fops);
389 390 391
	if (!pde)
		goto err2;

392
#ifdef CONFIG_IP_ROUTE_CLASSID
393
	pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
394 395 396 397 398
	if (!pde)
		goto err3;
#endif
	return 0;

399
#ifdef CONFIG_IP_ROUTE_CLASSID
400 401 402 403 404 405 406 407
err3:
	remove_proc_entry("rt_cache", net->proc_net_stat);
#endif
err2:
	remove_proc_entry("rt_cache", net->proc_net);
err1:
	return -ENOMEM;
}
408 409 410 411 412

static void __net_exit ip_rt_do_proc_exit(struct net *net)
{
	remove_proc_entry("rt_cache", net->proc_net_stat);
	remove_proc_entry("rt_cache", net->proc_net);
413
#ifdef CONFIG_IP_ROUTE_CLASSID
414
	remove_proc_entry("rt_acct", net->proc_net);
415
#endif
416 417 418 419 420 421 422 423 424 425 426 427
}

static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
	.init = ip_rt_do_proc_init,
	.exit = ip_rt_do_proc_exit,
};

static int __init ip_rt_proc_init(void)
{
	return register_pernet_subsys(&ip_rt_proc_ops);
}

428
#else
429
static inline int ip_rt_proc_init(void)
430 431 432
{
	return 0;
}
L
Linus Torvalds 已提交
433
#endif /* CONFIG_PROC_FS */
434

435
static inline bool rt_is_expired(const struct rtable *rth)
436
{
F
fan.du 已提交
437
	return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
438 439
}

440
void rt_cache_flush(struct net *net)
L
Linus Torvalds 已提交
441
{
F
fan.du 已提交
442
	rt_genid_bump_ipv4(net);
E
Eric Dumazet 已提交
443 444
}

445 446 447
static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
					   struct sk_buff *skb,
					   const void *daddr)
D
David Miller 已提交
448
{
449 450
	struct net_device *dev = dst->dev;
	const __be32 *pkey = daddr;
451
	const struct rtable *rt;
D
David Miller 已提交
452 453
	struct neighbour *n;

454
	rt = (const struct rtable *) dst;
455
	if (rt->rt_gateway)
456
		pkey = (const __be32 *) &rt->rt_gateway;
457 458
	else if (skb)
		pkey = &ip_hdr(skb)->daddr;
459

460
	n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
461 462
	if (n)
		return n;
463
	return neigh_create(&arp_tbl, pkey, dev);
464 465
}

466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
{
	struct net_device *dev = dst->dev;
	const __be32 *pkey = daddr;
	const struct rtable *rt;

	rt = (const struct rtable *)dst;
	if (rt->rt_gateway)
		pkey = (const __be32 *)&rt->rt_gateway;
	else if (!daddr ||
		 (rt->rt_flags &
		  (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
		return;

	__ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
}

483 484
#define IP_IDENTS_SZ 2048u

E
Eric Dumazet 已提交
485 486
static atomic_t *ip_idents __read_mostly;
static u32 *ip_tstamps __read_mostly;
487 488 489 490 491 492 493

/* In order to protect privacy, we add a perturbation to identifiers
 * if one generator is seldom used. This makes hard for an attacker
 * to infer how many packets were sent between two points in time.
 */
u32 ip_idents_reserve(u32 hash, int segs)
{
E
Eric Dumazet 已提交
494 495 496
	u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
	atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
	u32 old = ACCESS_ONCE(*p_tstamp);
497
	u32 now = (u32)jiffies;
498
	u32 new, delta = 0;
499

E
Eric Dumazet 已提交
500
	if (old != now && cmpxchg(p_tstamp, old, now) == old)
501 502
		delta = prandom_u32_max(now - old);

503 504 505 506 507 508 509
	/* Do not use atomic_add_return() as it makes UBSAN unhappy */
	do {
		old = (u32)atomic_read(p_id);
		new = old + delta + segs;
	} while (atomic_cmpxchg(p_id, old, new) != old);

	return new - segs;
510 511
}
EXPORT_SYMBOL(ip_idents_reserve);
L
Linus Torvalds 已提交
512

513
void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
L
Linus Torvalds 已提交
514
{
E
Eric Dumazet 已提交
515 516
	static u32 ip_idents_hashrnd __read_mostly;
	u32 hash, id;
L
Linus Torvalds 已提交
517

E
Eric Dumazet 已提交
518
	net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
L
Linus Torvalds 已提交
519

520 521
	hash = jhash_3words((__force u32)iph->daddr,
			    (__force u32)iph->saddr,
522
			    iph->protocol ^ net_hash_mix(net),
523
			    ip_idents_hashrnd);
E
Eric Dumazet 已提交
524 525
	id = ip_idents_reserve(hash, segs);
	iph->id = htons(id);
L
Linus Torvalds 已提交
526
}
E
Eric Dumazet 已提交
527
EXPORT_SYMBOL(__ip_select_ident);
L
Linus Torvalds 已提交
528

529 530
static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
			     const struct sock *sk,
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
			     const struct iphdr *iph,
			     int oif, u8 tos,
			     u8 prot, u32 mark, int flow_flags)
{
	if (sk) {
		const struct inet_sock *inet = inet_sk(sk);

		oif = sk->sk_bound_dev_if;
		mark = sk->sk_mark;
		tos = RT_CONN_FLAGS(sk);
		prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
	}
	flowi4_init_output(fl4, oif, mark, tos,
			   RT_SCOPE_UNIVERSE, prot,
			   flow_flags,
546 547
			   iph->daddr, iph->saddr, 0, 0,
			   sock_net_uid(net, sk));
548 549
}

E
Eric Dumazet 已提交
550 551
static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
			       const struct sock *sk)
552
{
553
	const struct net *net = dev_net(skb->dev);
554 555 556 557 558 559
	const struct iphdr *iph = ip_hdr(skb);
	int oif = skb->dev->ifindex;
	u8 tos = RT_TOS(iph->tos);
	u8 prot = iph->protocol;
	u32 mark = skb->mark;

560
	__build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
561 562
}

E
Eric Dumazet 已提交
563
static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
564 565
{
	const struct inet_sock *inet = inet_sk(sk);
E
Eric Dumazet 已提交
566
	const struct ip_options_rcu *inet_opt;
567 568 569 570 571 572 573 574 575 576
	__be32 daddr = inet->inet_daddr;

	rcu_read_lock();
	inet_opt = rcu_dereference(inet->inet_opt);
	if (inet_opt && inet_opt->opt.srr)
		daddr = inet_opt->opt.faddr;
	flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
			   RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
			   inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
			   inet_sk_flowi_flags(sk),
577
			   daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
578 579 580
	rcu_read_unlock();
}

E
Eric Dumazet 已提交
581 582
static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
				 const struct sk_buff *skb)
583 584 585 586 587 588 589
{
	if (skb)
		build_skb_flow_key(fl4, skb, sk);
	else
		build_sk_flow_key(fl4, sk);
}

590 591 592 593 594 595
static inline void rt_free(struct rtable *rt)
{
	call_rcu(&rt->dst.rcu_head, dst_rcu_free);
}

static DEFINE_SPINLOCK(fnhe_lock);
596

597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
{
	struct rtable *rt;

	rt = rcu_dereference(fnhe->fnhe_rth_input);
	if (rt) {
		RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
		rt_free(rt);
	}
	rt = rcu_dereference(fnhe->fnhe_rth_output);
	if (rt) {
		RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
		rt_free(rt);
	}
}

613
static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
614 615 616 617 618 619 620 621 622
{
	struct fib_nh_exception *fnhe, *oldest;

	oldest = rcu_dereference(hash->chain);
	for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
		if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
			oldest = fnhe;
	}
623
	fnhe_flush_routes(oldest);
624 625 626
	return oldest;
}

627 628
static inline u32 fnhe_hashfun(__be32 daddr)
{
E
Eric Dumazet 已提交
629
	static u32 fnhe_hashrnd __read_mostly;
630 631
	u32 hval;

E
Eric Dumazet 已提交
632 633 634
	net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
	hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
	return hash_32(hval, FNHE_HASH_SHIFT);
635 636
}

637 638 639 640 641 642 643 644 645 646 647 648
static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
{
	rt->rt_pmtu = fnhe->fnhe_pmtu;
	rt->dst.expires = fnhe->fnhe_expires;

	if (fnhe->fnhe_gw) {
		rt->rt_flags |= RTCF_REDIRECTED;
		rt->rt_gateway = fnhe->fnhe_gw;
		rt->rt_uses_gateway = 1;
	}
}

649 650
static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
				  u32 pmtu, unsigned long expires)
651
{
652
	struct fnhe_hash_bucket *hash;
653
	struct fib_nh_exception *fnhe;
654 655
	struct rtable *rt;
	unsigned int i;
656
	int depth;
657 658
	u32 hval = fnhe_hashfun(daddr);

659
	spin_lock_bh(&fnhe_lock);
660

661
	hash = rcu_dereference(nh->nh_exceptions);
662
	if (!hash) {
663
		hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
664
		if (!hash)
665
			goto out_unlock;
666
		rcu_assign_pointer(nh->nh_exceptions, hash);
667 668 669 670 671 672 673 674
	}

	hash += hval;

	depth = 0;
	for (fnhe = rcu_dereference(hash->chain); fnhe;
	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
		if (fnhe->fnhe_daddr == daddr)
675
			break;
676 677 678
		depth++;
	}

679 680 681 682 683
	if (fnhe) {
		if (gw)
			fnhe->fnhe_gw = gw;
		if (pmtu) {
			fnhe->fnhe_pmtu = pmtu;
684
			fnhe->fnhe_expires = max(1UL, expires);
685
		}
686
		/* Update all cached dsts too */
687 688 689 690
		rt = rcu_dereference(fnhe->fnhe_rth_input);
		if (rt)
			fill_route_from_fnhe(rt, fnhe);
		rt = rcu_dereference(fnhe->fnhe_rth_output);
691 692
		if (rt)
			fill_route_from_fnhe(rt, fnhe);
693 694 695 696 697 698 699 700 701 702 703
	} else {
		if (depth > FNHE_RECLAIM_DEPTH)
			fnhe = fnhe_oldest(hash);
		else {
			fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
			if (!fnhe)
				goto out_unlock;

			fnhe->fnhe_next = hash->chain;
			rcu_assign_pointer(hash->chain, fnhe);
		}
704
		fnhe->fnhe_genid = fnhe_genid(dev_net(nh->nh_dev));
705 706 707 708
		fnhe->fnhe_daddr = daddr;
		fnhe->fnhe_gw = gw;
		fnhe->fnhe_pmtu = pmtu;
		fnhe->fnhe_expires = expires;
709 710 711 712 713

		/* Exception created; mark the cached routes for the nexthop
		 * stale, so anyone caching it rechecks if this exception
		 * applies to them.
		 */
714 715 716 717
		rt = rcu_dereference(nh->nh_rth_input);
		if (rt)
			rt->dst.obsolete = DST_OBSOLETE_KILL;

718 719 720 721 722 723 724
		for_each_possible_cpu(i) {
			struct rtable __rcu **prt;
			prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
			rt = rcu_dereference(*prt);
			if (rt)
				rt->dst.obsolete = DST_OBSOLETE_KILL;
		}
725 726 727
	}

	fnhe->fnhe_stamp = jiffies;
728 729

out_unlock:
730
	spin_unlock_bh(&fnhe_lock);
731 732
}

733 734
static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
			     bool kill_route)
L
Linus Torvalds 已提交
735
{
736
	__be32 new_gw = icmp_hdr(skb)->un.gateway;
737
	__be32 old_gw = ip_hdr(skb)->saddr;
738 739
	struct net_device *dev = skb->dev;
	struct in_device *in_dev;
740
	struct fib_result res;
741
	struct neighbour *n;
742
	struct net *net;
L
Linus Torvalds 已提交
743

744 745 746 747 748 749 750 751 752 753 754
	switch (icmp_hdr(skb)->code & 7) {
	case ICMP_REDIR_NET:
	case ICMP_REDIR_NETTOS:
	case ICMP_REDIR_HOST:
	case ICMP_REDIR_HOSTTOS:
		break;

	default:
		return;
	}

755 756 757 758 759 760 761
	if (rt->rt_gateway != old_gw)
		return;

	in_dev = __in_dev_get_rcu(dev);
	if (!in_dev)
		return;

762
	net = dev_net(dev);
763 764 765
	if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
	    ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
	    ipv4_is_zeronet(new_gw))
L
Linus Torvalds 已提交
766 767 768 769 770 771 772 773
		goto reject_redirect;

	if (!IN_DEV_SHARED_MEDIA(in_dev)) {
		if (!inet_addr_onlink(in_dev, new_gw, old_gw))
			goto reject_redirect;
		if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
			goto reject_redirect;
	} else {
774
		if (inet_addr_type(net, new_gw) != RTN_UNICAST)
L
Linus Torvalds 已提交
775 776 777
			goto reject_redirect;
	}

778 779 780
	n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
	if (!n)
		n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
781
	if (!IS_ERR(n)) {
782 783 784
		if (!(n->nud_state & NUD_VALID)) {
			neigh_event_send(n, NULL);
		} else {
785
			if (fib_lookup(net, fl4, &res, 0) == 0) {
786 787
				struct fib_nh *nh = &FIB_RES_NH(res);

788
				update_or_create_fnhe(nh, fl4->daddr, new_gw,
789
						0, jiffies + ip_rt_gc_timeout);
790
			}
791 792
			if (kill_route)
				rt->dst.obsolete = DST_OBSOLETE_KILL;
793 794 795 796 797 798 799 800
			call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
		}
		neigh_release(n);
	}
	return;

reject_redirect:
#ifdef CONFIG_IP_ROUTE_VERBOSE
801 802 803 804 805
	if (IN_DEV_LOG_MARTIANS(in_dev)) {
		const struct iphdr *iph = (const struct iphdr *) skb->data;
		__be32 daddr = iph->daddr;
		__be32 saddr = iph->saddr;

806 807 808 809
		net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
				     "  Advised path = %pI4 -> %pI4\n",
				     &old_gw, dev->name, &new_gw,
				     &saddr, &daddr);
810
	}
811 812 813 814
#endif
	;
}

815 816 817 818
static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
{
	struct rtable *rt;
	struct flowi4 fl4;
819
	const struct iphdr *iph = (const struct iphdr *) skb->data;
820
	struct net *net = dev_net(skb->dev);
821 822 823 824
	int oif = skb->dev->ifindex;
	u8 tos = RT_TOS(iph->tos);
	u8 prot = iph->protocol;
	u32 mark = skb->mark;
825 826 827

	rt = (struct rtable *) dst;

828
	__build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
829
	__ip_do_redirect(rt, skb, &fl4, true);
830 831
}

L
Linus Torvalds 已提交
832 833
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
{
834
	struct rtable *rt = (struct rtable *)dst;
L
Linus Torvalds 已提交
835 836 837
	struct dst_entry *ret = dst;

	if (rt) {
T
Timo Teräs 已提交
838
		if (dst->obsolete > 0) {
L
Linus Torvalds 已提交
839 840
			ip_rt_put(rt);
			ret = NULL;
841 842
		} else if ((rt->rt_flags & RTCF_REDIRECTED) ||
			   rt->dst.expires) {
D
David S. Miller 已提交
843
			ip_rt_put(rt);
L
Linus Torvalds 已提交
844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
			ret = NULL;
		}
	}
	return ret;
}

/*
 * Algorithm:
 *	1. The first ip_rt_redirect_number redirects are sent
 *	   with exponential backoff, then we stop sending them at all,
 *	   assuming that the host ignores our redirects.
 *	2. If we did not see packets requiring redirects
 *	   during ip_rt_redirect_silence, we assume that the host
 *	   forgot redirected route and start to send redirects again.
 *
 * This algorithm is much cheaper and more intelligent than dumb load limiting
 * in icmp.c.
 *
 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 * and "frag. need" (breaks PMTU discovery) in icmp.c.
 */

void ip_rt_send_redirect(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
868
	struct rtable *rt = skb_rtable(skb);
869
	struct in_device *in_dev;
870
	struct inet_peer *peer;
871
	struct net *net;
872
	int log_martians;
873
	int vif;
L
Linus Torvalds 已提交
874

875
	rcu_read_lock();
876
	in_dev = __in_dev_get_rcu(rt->dst.dev);
877 878
	if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
		rcu_read_unlock();
L
Linus Torvalds 已提交
879
		return;
880 881
	}
	log_martians = IN_DEV_LOG_MARTIANS(in_dev);
882
	vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
883
	rcu_read_unlock();
L
Linus Torvalds 已提交
884

885
	net = dev_net(rt->dst.dev);
886
	peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
887
	if (!peer) {
J
Julian Anastasov 已提交
888 889
		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
			  rt_nexthop(rt, ip_hdr(skb)->daddr));
890 891 892
		return;
	}

L
Linus Torvalds 已提交
893 894 895
	/* No redirected packets during ip_rt_redirect_silence;
	 * reset the algorithm.
	 */
896 897
	if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
		peer->rate_tokens = 0;
L
Linus Torvalds 已提交
898 899

	/* Too many ignored redirects; do not send anything
900
	 * set dst.rate_last to the last seen redirected packet.
L
Linus Torvalds 已提交
901
	 */
902 903
	if (peer->rate_tokens >= ip_rt_redirect_number) {
		peer->rate_last = jiffies;
904
		goto out_put_peer;
L
Linus Torvalds 已提交
905 906 907 908 909
	}

	/* Check for load limit; set rate_last to the latest sent
	 * redirect.
	 */
910
	if (peer->rate_tokens == 0 ||
911
	    time_after(jiffies,
912 913
		       (peer->rate_last +
			(ip_rt_redirect_load << peer->rate_tokens)))) {
J
Julian Anastasov 已提交
914 915 916
		__be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);

		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
917 918
		peer->rate_last = jiffies;
		++peer->rate_tokens;
L
Linus Torvalds 已提交
919
#ifdef CONFIG_IP_ROUTE_VERBOSE
920
		if (log_martians &&
921 922
		    peer->rate_tokens == ip_rt_redirect_number)
			net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
923
					     &ip_hdr(skb)->saddr, inet_iif(skb),
J
Julian Anastasov 已提交
924
					     &ip_hdr(skb)->daddr, &gw);
L
Linus Torvalds 已提交
925 926
#endif
	}
927 928
out_put_peer:
	inet_putpeer(peer);
L
Linus Torvalds 已提交
929 930 931 932
}

static int ip_error(struct sk_buff *skb)
{
933
	struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
E
Eric Dumazet 已提交
934
	struct rtable *rt = skb_rtable(skb);
935
	struct inet_peer *peer;
L
Linus Torvalds 已提交
936
	unsigned long now;
937
	struct net *net;
938
	bool send;
L
Linus Torvalds 已提交
939 940
	int code;

941 942 943 944
	/* IP on this device is disabled. */
	if (!in_dev)
		goto out;

945 946 947 948
	net = dev_net(rt->dst.dev);
	if (!IN_DEV_FORWARD(in_dev)) {
		switch (rt->dst.error) {
		case EHOSTUNREACH:
E
Eric Dumazet 已提交
949
			__IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
950 951 952
			break;

		case ENETUNREACH:
E
Eric Dumazet 已提交
953
			__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
954 955 956 957 958
			break;
		}
		goto out;
	}

959
	switch (rt->dst.error) {
J
Joe Perches 已提交
960 961 962 963 964 965 966 967
	case EINVAL:
	default:
		goto out;
	case EHOSTUNREACH:
		code = ICMP_HOST_UNREACH;
		break;
	case ENETUNREACH:
		code = ICMP_NET_UNREACH;
E
Eric Dumazet 已提交
968
		__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
J
Joe Perches 已提交
969 970 971 972
		break;
	case EACCES:
		code = ICMP_PKT_FILTERED;
		break;
L
Linus Torvalds 已提交
973 974
	}

975
	peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
976
			       l3mdev_master_ifindex(skb->dev), 1);
977 978 979 980 981 982 983 984 985 986 987 988

	send = true;
	if (peer) {
		now = jiffies;
		peer->rate_tokens += now - peer->rate_last;
		if (peer->rate_tokens > ip_rt_error_burst)
			peer->rate_tokens = ip_rt_error_burst;
		peer->rate_last = now;
		if (peer->rate_tokens >= ip_rt_error_cost)
			peer->rate_tokens -= ip_rt_error_cost;
		else
			send = false;
989
		inet_putpeer(peer);
L
Linus Torvalds 已提交
990
	}
991 992
	if (send)
		icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
L
Linus Torvalds 已提交
993 994 995

out:	kfree_skb(skb);
	return 0;
996
}
L
Linus Torvalds 已提交
997

998
static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
L
Linus Torvalds 已提交
999
{
1000
	struct dst_entry *dst = &rt->dst;
1001
	struct fib_result res;
1002

1003 1004 1005
	if (dst_metric_locked(dst, RTAX_MTU))
		return;

1006
	if (ipv4_mtu(dst) < mtu)
1007 1008
		return;

1009 1010
	if (mtu < ip_rt_min_pmtu)
		mtu = ip_rt_min_pmtu;
1011

1012 1013 1014 1015
	if (rt->rt_pmtu == mtu &&
	    time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
		return;

1016
	rcu_read_lock();
1017
	if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1018 1019
		struct fib_nh *nh = &FIB_RES_NH(res);

1020 1021
		update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
				      jiffies + ip_rt_mtu_expires);
1022
	}
1023
	rcu_read_unlock();
L
Linus Torvalds 已提交
1024 1025
}

1026 1027 1028 1029 1030 1031 1032
static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
			      struct sk_buff *skb, u32 mtu)
{
	struct rtable *rt = (struct rtable *) dst;
	struct flowi4 fl4;

	ip_rt_build_flow_key(&fl4, sk, skb);
1033
	__ip_rt_update_pmtu(rt, &fl4, mtu);
1034 1035
}

1036 1037 1038
void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
		      int oif, u32 mark, u8 protocol, int flow_flags)
{
1039
	const struct iphdr *iph = (const struct iphdr *) skb->data;
1040 1041 1042
	struct flowi4 fl4;
	struct rtable *rt;

1043 1044 1045
	if (!mark)
		mark = IP4_REPLY_MARK(net, skb->mark);

1046
	__build_flow_key(net, &fl4, NULL, iph, oif,
1047
			 RT_TOS(iph->tos), protocol, mark, flow_flags);
1048 1049
	rt = __ip_route_output_key(net, &fl4);
	if (!IS_ERR(rt)) {
1050
		__ip_rt_update_pmtu(rt, &fl4, mtu);
1051 1052 1053 1054 1055
		ip_rt_put(rt);
	}
}
EXPORT_SYMBOL_GPL(ipv4_update_pmtu);

1056
static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1057
{
1058 1059 1060
	const struct iphdr *iph = (const struct iphdr *) skb->data;
	struct flowi4 fl4;
	struct rtable *rt;
1061

1062
	__build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1063 1064 1065 1066

	if (!fl4.flowi4_mark)
		fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);

1067 1068 1069 1070 1071
	rt = __ip_route_output_key(sock_net(sk), &fl4);
	if (!IS_ERR(rt)) {
		__ip_rt_update_pmtu(rt, &fl4, mtu);
		ip_rt_put(rt);
	}
1072
}
1073 1074 1075 1076 1077 1078

void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
{
	const struct iphdr *iph = (const struct iphdr *) skb->data;
	struct flowi4 fl4;
	struct rtable *rt;
1079
	struct dst_entry *odst = NULL;
1080
	bool new = false;
1081
	struct net *net = sock_net(sk);
1082 1083

	bh_lock_sock(sk);
1084 1085 1086 1087

	if (!ip_sk_accept_pmtu(sk))
		goto out;

1088
	odst = sk_dst_get(sk);
1089

1090
	if (sock_owned_by_user(sk) || !odst) {
1091 1092 1093 1094
		__ipv4_sk_update_pmtu(skb, sk, mtu);
		goto out;
	}

1095
	__build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1096

1097
	rt = (struct rtable *)odst;
1098
	if (odst->obsolete && !odst->ops->check(odst, 0)) {
1099 1100 1101
		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
		if (IS_ERR(rt))
			goto out;
1102 1103

		new = true;
1104 1105 1106 1107
	}

	__ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);

1108
	if (!dst_check(&rt->dst, 0)) {
1109 1110 1111
		if (new)
			dst_release(&rt->dst);

1112 1113 1114 1115
		rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
		if (IS_ERR(rt))
			goto out;

1116
		new = true;
1117 1118
	}

1119
	if (new)
1120
		sk_dst_set(sk, &rt->dst);
1121 1122 1123

out:
	bh_unlock_sock(sk);
1124
	dst_release(odst);
1125
}
1126
EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1127

1128 1129 1130
void ipv4_redirect(struct sk_buff *skb, struct net *net,
		   int oif, u32 mark, u8 protocol, int flow_flags)
{
1131
	const struct iphdr *iph = (const struct iphdr *) skb->data;
1132 1133 1134
	struct flowi4 fl4;
	struct rtable *rt;

1135
	__build_flow_key(net, &fl4, NULL, iph, oif,
1136
			 RT_TOS(iph->tos), protocol, mark, flow_flags);
1137 1138
	rt = __ip_route_output_key(net, &fl4);
	if (!IS_ERR(rt)) {
1139
		__ip_do_redirect(rt, skb, &fl4, false);
1140 1141 1142 1143 1144 1145 1146
		ip_rt_put(rt);
	}
}
EXPORT_SYMBOL_GPL(ipv4_redirect);

void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
{
1147 1148 1149
	const struct iphdr *iph = (const struct iphdr *) skb->data;
	struct flowi4 fl4;
	struct rtable *rt;
1150
	struct net *net = sock_net(sk);
1151

1152 1153
	__build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
	rt = __ip_route_output_key(net, &fl4);
1154
	if (!IS_ERR(rt)) {
1155
		__ip_do_redirect(rt, skb, &fl4, false);
1156 1157
		ip_rt_put(rt);
	}
1158 1159 1160
}
EXPORT_SYMBOL_GPL(ipv4_sk_redirect);

1161 1162 1163 1164
static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
{
	struct rtable *rt = (struct rtable *) dst;

1165 1166 1167 1168
	/* All IPV4 dsts are created with ->obsolete set to the value
	 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
	 * into this function always.
	 *
1169 1170 1171
	 * When a PMTU/redirect information update invalidates a route,
	 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
	 * DST_OBSOLETE_DEAD by dst_free().
1172
	 */
1173
	if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1174
		return NULL;
T
Timo Teräs 已提交
1175
	return dst;
L
Linus Torvalds 已提交
1176 1177 1178 1179 1180 1181 1182 1183
}

static void ipv4_link_failure(struct sk_buff *skb)
{
	struct rtable *rt;

	icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);

E
Eric Dumazet 已提交
1184
	rt = skb_rtable(skb);
1185 1186
	if (rt)
		dst_set_expires(&rt->dst, 0);
L
Linus Torvalds 已提交
1187 1188
}

E
Eric W. Biederman 已提交
1189
static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
1190
{
1191 1192 1193
	pr_debug("%s: %pI4 -> %pI4, %s\n",
		 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
		 skb->dev ? skb->dev->name : "?");
L
Linus Torvalds 已提交
1194
	kfree_skb(skb);
1195
	WARN_ON(1);
L
Linus Torvalds 已提交
1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
	return 0;
}

/*
   We do not cache source address of outgoing interface,
   because it is used only by IP RR, TS and SRR options,
   so that it out of fast path.

   BTW remember: "addr" is allowed to be not aligned
   in IP options!
 */

1208
void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
L
Linus Torvalds 已提交
1209
{
A
Al Viro 已提交
1210
	__be32 src;
L
Linus Torvalds 已提交
1211

1212
	if (rt_is_output_route(rt))
1213
		src = ip_hdr(skb)->saddr;
E
Eric Dumazet 已提交
1214
	else {
1215 1216 1217 1218 1219 1220 1221 1222 1223
		struct fib_result res;
		struct flowi4 fl4;
		struct iphdr *iph;

		iph = ip_hdr(skb);

		memset(&fl4, 0, sizeof(fl4));
		fl4.daddr = iph->daddr;
		fl4.saddr = iph->saddr;
1224
		fl4.flowi4_tos = RT_TOS(iph->tos);
1225 1226 1227
		fl4.flowi4_oif = rt->dst.dev->ifindex;
		fl4.flowi4_iif = skb->dev->ifindex;
		fl4.flowi4_mark = skb->mark;
1228

E
Eric Dumazet 已提交
1229
		rcu_read_lock();
1230
		if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1231
			src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
E
Eric Dumazet 已提交
1232
		else
1233 1234 1235
			src = inet_select_addr(rt->dst.dev,
					       rt_nexthop(rt, iph->daddr),
					       RT_SCOPE_UNIVERSE);
E
Eric Dumazet 已提交
1236 1237
		rcu_read_unlock();
	}
L
Linus Torvalds 已提交
1238 1239 1240
	memcpy(addr, &src, 4);
}

1241
#ifdef CONFIG_IP_ROUTE_CLASSID
L
Linus Torvalds 已提交
1242 1243
static void set_class_tag(struct rtable *rt, u32 tag)
{
1244 1245 1246 1247
	if (!(rt->dst.tclassid & 0xFFFF))
		rt->dst.tclassid |= tag & 0xFFFF;
	if (!(rt->dst.tclassid & 0xFFFF0000))
		rt->dst.tclassid |= tag & 0xFFFF0000;
L
Linus Torvalds 已提交
1248 1249 1250
}
#endif

1251 1252
static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
{
1253 1254 1255
	unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
	unsigned int advmss = max_t(unsigned int, dst->dev->mtu - header_size,
				    ip_rt_min_advmss);
1256

1257
	return min(advmss, IPV4_MAX_PMTU - header_size);
1258 1259
}

1260
static unsigned int ipv4_mtu(const struct dst_entry *dst)
1261
{
1262
	const struct rtable *rt = (const struct rtable *) dst;
1263 1264
	unsigned int mtu = rt->rt_pmtu;

1265
	if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1266
		mtu = dst_metric_raw(dst, RTAX_MTU);
1267

1268
	if (mtu)
1269 1270 1271
		return mtu;

	mtu = dst->dev->mtu;
1272 1273

	if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
J
Julian Anastasov 已提交
1274
		if (rt->rt_uses_gateway && mtu > 576)
1275 1276 1277
			mtu = 576;
	}

1278 1279 1280
	mtu = min_t(unsigned int, mtu, IP_MAX_MTU);

	return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1281 1282
}

1283
static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1284
{
1285
	struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1286 1287 1288
	struct fib_nh_exception *fnhe;
	u32 hval;

1289 1290 1291
	if (!hash)
		return NULL;

1292
	hval = fnhe_hashfun(daddr);
1293 1294 1295

	for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
	     fnhe = rcu_dereference(fnhe->fnhe_next)) {
1296 1297 1298 1299 1300
		if (fnhe->fnhe_daddr == daddr)
			return fnhe;
	}
	return NULL;
}
1301

1302
static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1303 1304
			      __be32 daddr)
{
1305 1306
	bool ret = false;

1307
	spin_lock_bh(&fnhe_lock);
1308

1309
	if (daddr == fnhe->fnhe_daddr) {
1310 1311
		struct rtable __rcu **porig;
		struct rtable *orig;
1312
		int genid = fnhe_genid(dev_net(rt->dst.dev));
1313 1314 1315 1316 1317 1318

		if (rt_is_input_route(rt))
			porig = &fnhe->fnhe_rth_input;
		else
			porig = &fnhe->fnhe_rth_output;
		orig = rcu_dereference(*porig);
1319 1320 1321

		if (fnhe->fnhe_genid != genid) {
			fnhe->fnhe_genid = genid;
1322 1323 1324
			fnhe->fnhe_gw = 0;
			fnhe->fnhe_pmtu = 0;
			fnhe->fnhe_expires = 0;
1325 1326
			fnhe_flush_routes(fnhe);
			orig = NULL;
1327
		}
1328 1329
		fill_route_from_fnhe(rt, fnhe);
		if (!rt->rt_gateway)
J
Julian Anastasov 已提交
1330
			rt->rt_gateway = daddr;
1331

1332 1333 1334 1335 1336 1337
		if (!(rt->dst.flags & DST_NOCACHE)) {
			rcu_assign_pointer(*porig, rt);
			if (orig)
				rt_free(orig);
			ret = true;
		}
1338 1339 1340 1341

		fnhe->fnhe_stamp = jiffies;
	}
	spin_unlock_bh(&fnhe_lock);
1342 1343

	return ret;
1344 1345
}

1346
static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1347
{
E
Eric Dumazet 已提交
1348
	struct rtable *orig, *prev, **p;
1349
	bool ret = true;
1350

E
Eric Dumazet 已提交
1351
	if (rt_is_input_route(rt)) {
1352
		p = (struct rtable **)&nh->nh_rth_input;
E
Eric Dumazet 已提交
1353
	} else {
1354
		p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
E
Eric Dumazet 已提交
1355
	}
1356 1357 1358 1359 1360
	orig = *p;

	prev = cmpxchg(p, orig, rt);
	if (prev == orig) {
		if (orig)
1361
			rt_free(orig);
J
Julian Anastasov 已提交
1362
	} else
1363 1364 1365 1366 1367
		ret = false;

	return ret;
}

E
Eric Dumazet 已提交
1368 1369 1370 1371 1372 1373
struct uncached_list {
	spinlock_t		lock;
	struct list_head	head;
};

static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1374 1375 1376

static void rt_add_uncached_list(struct rtable *rt)
{
E
Eric Dumazet 已提交
1377 1378 1379 1380 1381 1382 1383
	struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);

	rt->rt_uncached_list = ul;

	spin_lock_bh(&ul->lock);
	list_add_tail(&rt->rt_uncached, &ul->head);
	spin_unlock_bh(&ul->lock);
1384 1385 1386 1387 1388 1389
}

static void ipv4_dst_destroy(struct dst_entry *dst)
{
	struct rtable *rt = (struct rtable *) dst;

1390
	if (!list_empty(&rt->rt_uncached)) {
E
Eric Dumazet 已提交
1391 1392 1393
		struct uncached_list *ul = rt->rt_uncached_list;

		spin_lock_bh(&ul->lock);
1394
		list_del(&rt->rt_uncached);
E
Eric Dumazet 已提交
1395
		spin_unlock_bh(&ul->lock);
1396 1397 1398 1399 1400
	}
}

void rt_flush_dev(struct net_device *dev)
{
E
Eric Dumazet 已提交
1401 1402 1403 1404 1405 1406
	struct net *net = dev_net(dev);
	struct rtable *rt;
	int cpu;

	for_each_possible_cpu(cpu) {
		struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1407

E
Eric Dumazet 已提交
1408 1409
		spin_lock_bh(&ul->lock);
		list_for_each_entry(rt, &ul->head, rt_uncached) {
1410 1411 1412 1413 1414 1415
			if (rt->dst.dev != dev)
				continue;
			rt->dst.dev = net->loopback_dev;
			dev_hold(rt->dst.dev);
			dev_put(dev);
		}
E
Eric Dumazet 已提交
1416
		spin_unlock_bh(&ul->lock);
1417 1418 1419
	}
}

1420
static bool rt_cache_valid(const struct rtable *rt)
1421
{
1422 1423 1424
	return	rt &&
		rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
		!rt_is_expired(rt);
1425 1426
}

1427
static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1428
			   const struct fib_result *res,
1429
			   struct fib_nh_exception *fnhe,
1430
			   struct fib_info *fi, u16 type, u32 itag)
L
Linus Torvalds 已提交
1431
{
1432 1433
	bool cached = false;

L
Linus Torvalds 已提交
1434
	if (fi) {
1435 1436
		struct fib_nh *nh = &FIB_RES_NH(*res);

J
Julian Anastasov 已提交
1437
		if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1438
			rt->rt_gateway = nh->nh_gw;
J
Julian Anastasov 已提交
1439 1440
			rt->rt_uses_gateway = 1;
		}
D
David S. Miller 已提交
1441
		dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1442
#ifdef CONFIG_IP_ROUTE_CLASSID
1443
		rt->dst.tclassid = nh->nh_tclassid;
L
Linus Torvalds 已提交
1444
#endif
1445
		rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1446
		if (unlikely(fnhe))
1447
			cached = rt_bind_exception(rt, fnhe, daddr);
1448
		else if (!(rt->dst.flags & DST_NOCACHE))
1449
			cached = rt_cache_route(nh, rt);
J
Julian Anastasov 已提交
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
		if (unlikely(!cached)) {
			/* Routes we intend to cache in nexthop exception or
			 * FIB nexthop have the DST_NOCACHE bit clear.
			 * However, if we are unsuccessful at storing this
			 * route into the cache we really need to set it.
			 */
			rt->dst.flags |= DST_NOCACHE;
			if (!rt->rt_gateway)
				rt->rt_gateway = daddr;
			rt_add_uncached_list(rt);
		}
	} else
1462
		rt_add_uncached_list(rt);
1463

1464
#ifdef CONFIG_IP_ROUTE_CLASSID
L
Linus Torvalds 已提交
1465
#ifdef CONFIG_IP_MULTIPLE_TABLES
1466
	set_class_tag(rt, res->tclassid);
L
Linus Torvalds 已提交
1467 1468 1469 1470 1471
#endif
	set_class_tag(rt, itag);
#endif
}

1472 1473 1474
struct rtable *rt_dst_alloc(struct net_device *dev,
			    unsigned int flags, u16 type,
			    bool nopolicy, bool noxfrm, bool will_cache)
1475
{
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
	struct rtable *rt;

	rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
		       (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
		       (nopolicy ? DST_NOPOLICY : 0) |
		       (noxfrm ? DST_NOXFRM : 0));

	if (rt) {
		rt->rt_genid = rt_genid_ipv4(dev_net(dev));
		rt->rt_flags = flags;
		rt->rt_type = type;
		rt->rt_is_input = 0;
		rt->rt_iif = 0;
		rt->rt_pmtu = 0;
		rt->rt_gateway = 0;
		rt->rt_uses_gateway = 0;
D
David Ahern 已提交
1492
		rt->rt_table_id = 0;
1493 1494 1495 1496 1497 1498 1499 1500
		INIT_LIST_HEAD(&rt->rt_uncached);

		rt->dst.output = ip_output;
		if (flags & RTCF_LOCAL)
			rt->dst.input = ip_local_deliver;
	}

	return rt;
1501
}
1502
EXPORT_SYMBOL(rt_dst_alloc);
1503

1504
/* called in rcu_read_lock() section */
A
Al Viro 已提交
1505
static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
L
Linus Torvalds 已提交
1506 1507 1508
				u8 tos, struct net_device *dev, int our)
{
	struct rtable *rth;
1509
	struct in_device *in_dev = __in_dev_get_rcu(dev);
1510
	unsigned int flags = RTCF_MULTICAST;
L
Linus Torvalds 已提交
1511
	u32 itag = 0;
1512
	int err;
L
Linus Torvalds 已提交
1513 1514 1515

	/* Primary sanity checks. */

1516
	if (!in_dev)
L
Linus Torvalds 已提交
1517 1518
		return -EINVAL;

1519
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1520
	    skb->protocol != htons(ETH_P_IP))
L
Linus Torvalds 已提交
1521 1522
		goto e_inval;

1523 1524
	if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
		goto e_inval;
1525

1526 1527
	if (ipv4_is_zeronet(saddr)) {
		if (!ipv4_is_local_multicast(daddr))
L
Linus Torvalds 已提交
1528
			goto e_inval;
1529
	} else {
1530 1531
		err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
					  in_dev, &itag);
1532 1533 1534
		if (err < 0)
			goto e_err;
	}
1535 1536 1537 1538
	if (our)
		flags |= RTCF_LOCAL;

	rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1539
			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
L
Linus Torvalds 已提交
1540 1541 1542
	if (!rth)
		goto e_nobufs;

1543 1544 1545
#ifdef CONFIG_IP_ROUTE_CLASSID
	rth->dst.tclassid = itag;
#endif
1546
	rth->dst.output = ip_rt_bug;
1547
	rth->rt_is_input= 1;
L
Linus Torvalds 已提交
1548 1549

#ifdef CONFIG_IP_MROUTE
1550
	if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1551
		rth->dst.input = ip_mr_input;
L
Linus Torvalds 已提交
1552 1553 1554
#endif
	RT_CACHE_STAT_INC(in_slow_mc);

D
David S. Miller 已提交
1555 1556
	skb_dst_set(skb, &rth->dst);
	return 0;
L
Linus Torvalds 已提交
1557 1558 1559 1560

e_nobufs:
	return -ENOBUFS;
e_inval:
1561
	return -EINVAL;
1562 1563
e_err:
	return err;
L
Linus Torvalds 已提交
1564 1565 1566 1567 1568 1569
}


static void ip_handle_martian_source(struct net_device *dev,
				     struct in_device *in_dev,
				     struct sk_buff *skb,
A
Al Viro 已提交
1570 1571
				     __be32 daddr,
				     __be32 saddr)
L
Linus Torvalds 已提交
1572 1573 1574 1575 1576 1577 1578 1579
{
	RT_CACHE_STAT_INC(in_martian_src);
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
		/*
		 *	RFC1812 recommendation, if source is martian,
		 *	the only hint is MAC header.
		 */
J
Joe Perches 已提交
1580
		pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1581
			&daddr, &saddr, dev->name);
1582
		if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
J
Joe Perches 已提交
1583 1584 1585 1586
			print_hex_dump(KERN_WARNING, "ll header: ",
				       DUMP_PREFIX_OFFSET, 16, 1,
				       skb_mac_header(skb),
				       dev->hard_header_len, true);
L
Linus Torvalds 已提交
1587 1588 1589 1590 1591
		}
	}
#endif
}

1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
{
	struct fnhe_hash_bucket *hash;
	struct fib_nh_exception *fnhe, __rcu **fnhe_p;
	u32 hval = fnhe_hashfun(daddr);

	spin_lock_bh(&fnhe_lock);

	hash = rcu_dereference_protected(nh->nh_exceptions,
					 lockdep_is_held(&fnhe_lock));
	hash += hval;

	fnhe_p = &hash->chain;
	fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
	while (fnhe) {
		if (fnhe->fnhe_daddr == daddr) {
			rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
				fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
			fnhe_flush_routes(fnhe);
			kfree_rcu(fnhe, rcu);
			break;
		}
		fnhe_p = &fnhe->fnhe_next;
		fnhe = rcu_dereference_protected(fnhe->fnhe_next,
						 lockdep_is_held(&fnhe_lock));
	}

	spin_unlock_bh(&fnhe_lock);
}

1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
static void set_lwt_redirect(struct rtable *rth)
{
	if (lwtunnel_output_redirect(rth->dst.lwtstate)) {
		rth->dst.lwtstate->orig_output = rth->dst.output;
		rth->dst.output = lwtunnel_output;
	}

	if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
		rth->dst.lwtstate->orig_input = rth->dst.input;
		rth->dst.input = lwtunnel_input;
	}
}

1635
/* called in rcu_read_lock() section */
S
Stephen Hemminger 已提交
1636
static int __mkroute_input(struct sk_buff *skb,
1637
			   const struct fib_result *res,
S
Stephen Hemminger 已提交
1638
			   struct in_device *in_dev,
1639
			   __be32 daddr, __be32 saddr, u32 tos)
L
Linus Torvalds 已提交
1640
{
1641
	struct fib_nh_exception *fnhe;
L
Linus Torvalds 已提交
1642 1643 1644
	struct rtable *rth;
	int err;
	struct in_device *out_dev;
1645
	bool do_cache;
1646
	u32 itag = 0;
L
Linus Torvalds 已提交
1647 1648

	/* get a working reference to the output device */
1649
	out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1650
	if (!out_dev) {
1651
		net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
L
Linus Torvalds 已提交
1652 1653 1654
		return -EINVAL;
	}

1655
	err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1656
				  in_dev->dev, in_dev, &itag);
L
Linus Torvalds 已提交
1657
	if (err < 0) {
1658
		ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
L
Linus Torvalds 已提交
1659
					 saddr);
1660

L
Linus Torvalds 已提交
1661 1662 1663
		goto cleanup;
	}

J
Julian Anastasov 已提交
1664 1665
	do_cache = res->fi && !itag;
	if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1666
	    skb->protocol == htons(ETH_P_IP) &&
L
Linus Torvalds 已提交
1667
	    (IN_DEV_SHARED_MEDIA(out_dev) ||
1668 1669
	     inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
		IPCB(skb)->flags |= IPSKB_DOREDIRECT;
L
Linus Torvalds 已提交
1670 1671 1672 1673

	if (skb->protocol != htons(ETH_P_IP)) {
		/* Not IP (i.e. ARP). Do not create route, if it is
		 * invalid for proxy arp. DNAT routes are always valid.
1674 1675 1676 1677
		 *
		 * Proxy arp feature have been extended to allow, ARP
		 * replies back to the same interface, to support
		 * Private VLAN switch technologies. See arp.c.
L
Linus Torvalds 已提交
1678
		 */
1679 1680
		if (out_dev == in_dev &&
		    IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
L
Linus Torvalds 已提交
1681 1682 1683 1684 1685
			err = -EINVAL;
			goto cleanup;
		}
	}

1686
	fnhe = find_exception(&FIB_RES_NH(*res), daddr);
J
Julian Anastasov 已提交
1687
	if (do_cache) {
1688
		if (fnhe) {
1689
			rth = rcu_dereference(fnhe->fnhe_rth_input);
1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
			if (rth && rth->dst.expires &&
			    time_after(jiffies, rth->dst.expires)) {
				ip_del_fnhe(&FIB_RES_NH(*res), daddr);
				fnhe = NULL;
			} else {
				goto rt_cache;
			}
		}

		rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1700

1701
rt_cache:
J
Julian Anastasov 已提交
1702 1703 1704
		if (rt_cache_valid(rth)) {
			skb_dst_set_noref(skb, &rth->dst);
			goto out;
1705 1706
		}
	}
1707

1708
	rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1709
			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
1710
			   IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
L
Linus Torvalds 已提交
1711 1712 1713 1714 1715
	if (!rth) {
		err = -ENOBUFS;
		goto cleanup;
	}

1716
	rth->rt_is_input = 1;
D
David Ahern 已提交
1717 1718
	if (res->table)
		rth->rt_table_id = res->table->tb_id;
D
Duan Jiong 已提交
1719
	RT_CACHE_STAT_INC(in_slow_tot);
L
Linus Torvalds 已提交
1720

1721
	rth->dst.input = ip_forward;
L
Linus Torvalds 已提交
1722

1723
	rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag);
1724
	set_lwt_redirect(rth);
1725
	skb_dst_set(skb, &rth->dst);
1726
out:
L
Linus Torvalds 已提交
1727 1728 1729
	err = 0;
 cleanup:
	return err;
1730
}
L
Linus Torvalds 已提交
1731

1732 1733
#ifdef CONFIG_IP_ROUTE_MULTIPATH
/* To make ICMP packets follow the right flow, the multipath hash is
1734
 * calculated from the inner IP addresses.
1735
 */
1736 1737
static void ip_multipath_l3_keys(const struct sk_buff *skb,
				 struct flow_keys *hash_keys)
1738 1739
{
	const struct iphdr *outer_iph = ip_hdr(skb);
1740
	const struct iphdr *inner_iph;
1741 1742
	const struct icmphdr *icmph;
	struct iphdr _inner_iph;
1743 1744 1745 1746 1747 1748
	struct icmphdr _icmph;

	hash_keys->addrs.v4addrs.src = outer_iph->saddr;
	hash_keys->addrs.v4addrs.dst = outer_iph->daddr;
	if (likely(outer_iph->protocol != IPPROTO_ICMP))
		return;
1749 1750

	if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1751
		return;
1752 1753 1754 1755

	icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
				   &_icmph);
	if (!icmph)
1756
		return;
1757 1758 1759 1760

	if (icmph->type != ICMP_DEST_UNREACH &&
	    icmph->type != ICMP_REDIRECT &&
	    icmph->type != ICMP_TIME_EXCEEDED &&
1761 1762
	    icmph->type != ICMP_PARAMETERPROB)
		return;
1763 1764 1765 1766 1767

	inner_iph = skb_header_pointer(skb,
				       outer_iph->ihl * 4 + sizeof(_icmph),
				       sizeof(_inner_iph), &_inner_iph);
	if (!inner_iph)
1768 1769 1770 1771
		return;
	hash_keys->addrs.v4addrs.src = inner_iph->saddr;
	hash_keys->addrs.v4addrs.dst = inner_iph->daddr;
}
1772

1773 1774 1775 1776 1777 1778 1779
/* if skb is set it will be used and fl4 can be NULL */
int fib_multipath_hash(const struct fib_info *fi, const struct flowi4 *fl4,
		       const struct sk_buff *skb)
{
	struct net *net = fi->fib_net;
	struct flow_keys hash_keys;
	u32 mhash;
1780

1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
	case 0:
		memset(&hash_keys, 0, sizeof(hash_keys));
		hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
		if (skb) {
			ip_multipath_l3_keys(skb, &hash_keys);
		} else {
			hash_keys.addrs.v4addrs.src = fl4->saddr;
			hash_keys.addrs.v4addrs.dst = fl4->daddr;
		}
		break;
	case 1:
		/* skb is currently provided only when forwarding */
		if (skb) {
			unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
			struct flow_keys keys;

			/* short-circuit if we already have L4 hash present */
			if (skb->l4_hash)
				return skb_get_hash_raw(skb) >> 1;
			memset(&hash_keys, 0, sizeof(hash_keys));
			skb_flow_dissect_flow_keys(skb, &keys, flag);
			hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
			hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
			hash_keys.ports.src = keys.ports.src;
			hash_keys.ports.dst = keys.ports.dst;
			hash_keys.basic.ip_proto = keys.basic.ip_proto;
		} else {
			memset(&hash_keys, 0, sizeof(hash_keys));
			hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
			hash_keys.addrs.v4addrs.src = fl4->saddr;
			hash_keys.addrs.v4addrs.dst = fl4->daddr;
			hash_keys.ports.src = fl4->fl4_sport;
			hash_keys.ports.dst = fl4->fl4_dport;
			hash_keys.basic.ip_proto = fl4->flowi4_proto;
		}
		break;
	}
	mhash = flow_hash_from_keys(&hash_keys);
1820

1821 1822 1823
	return mhash >> 1;
}
EXPORT_SYMBOL_GPL(fib_multipath_hash);
1824 1825
#endif /* CONFIG_IP_ROUTE_MULTIPATH */

S
Stephen Hemminger 已提交
1826 1827 1828 1829
static int ip_mkroute_input(struct sk_buff *skb,
			    struct fib_result *res,
			    struct in_device *in_dev,
			    __be32 daddr, __be32 saddr, u32 tos)
L
Linus Torvalds 已提交
1830 1831
{
#ifdef CONFIG_IP_ROUTE_MULTIPATH
P
Peter Nørlund 已提交
1832
	if (res->fi && res->fi->fib_nhs > 1) {
1833
		int h = fib_multipath_hash(res->fi, NULL, skb);
P
Peter Nørlund 已提交
1834 1835 1836

		fib_select_multipath(res, h);
	}
L
Linus Torvalds 已提交
1837 1838 1839
#endif

	/* create a routing cache entry */
1840
	return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
L
Linus Torvalds 已提交
1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
}

/*
 *	NOTE. We drop all the packets that has local source
 *	addresses, because every properly looped back packet
 *	must have correct destination already attached by output routine.
 *
 *	Such approach solves two big problems:
 *	1. Not simplex devices are handled properly.
 *	2. IP spoofing attempts are filtered with 100% of guarantee.
E
Eric Dumazet 已提交
1851
 *	called with rcu_read_lock()
L
Linus Torvalds 已提交
1852 1853
 */

A
Al Viro 已提交
1854
static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1855 1856
			       u8 tos, struct net_device *dev,
			       struct fib_result *res)
L
Linus Torvalds 已提交
1857
{
1858
	struct in_device *in_dev = __in_dev_get_rcu(dev);
1859
	struct ip_tunnel_info *tun_info;
1860
	struct flowi4	fl4;
1861
	unsigned int	flags = 0;
L
Linus Torvalds 已提交
1862
	u32		itag = 0;
1863
	struct rtable	*rth;
L
Linus Torvalds 已提交
1864
	int		err = -EINVAL;
D
Daniel Baluta 已提交
1865
	struct net    *net = dev_net(dev);
1866
	bool do_cache;
L
Linus Torvalds 已提交
1867 1868 1869 1870 1871 1872 1873 1874 1875 1876

	/* IP on this device is disabled. */

	if (!in_dev)
		goto out;

	/* Check for the most weird martians, which can be not detected
	   by fib_lookup.
	 */

1877
	tun_info = skb_tunnel_info(skb);
1878
	if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1879 1880 1881
		fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
	else
		fl4.flowi4_tun_key.tun_id = 0;
T
Thomas Graf 已提交
1882 1883
	skb_dst_drop(skb);

1884
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
L
Linus Torvalds 已提交
1885 1886
		goto martian_source;

1887 1888
	res->fi = NULL;
	res->table = NULL;
1889
	if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
L
Linus Torvalds 已提交
1890 1891 1892 1893 1894
		goto brd_input;

	/* Accept zero addresses only to limited broadcast;
	 * I even do not know to fix it or not. Waiting for complains :-)
	 */
1895
	if (ipv4_is_zeronet(saddr))
L
Linus Torvalds 已提交
1896 1897
		goto martian_source;

1898
	if (ipv4_is_zeronet(daddr))
L
Linus Torvalds 已提交
1899 1900
		goto martian_destination;

1901 1902 1903 1904 1905
	/* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
	 * and call it once if daddr or/and saddr are loopback addresses
	 */
	if (ipv4_is_loopback(daddr)) {
		if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1906
			goto martian_destination;
1907 1908
	} else if (ipv4_is_loopback(saddr)) {
		if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1909 1910 1911
			goto martian_source;
	}

L
Linus Torvalds 已提交
1912 1913 1914
	/*
	 *	Now we are ready to route packet.
	 */
1915
	fl4.flowi4_oif = 0;
1916
	fl4.flowi4_iif = dev->ifindex;
1917 1918 1919
	fl4.flowi4_mark = skb->mark;
	fl4.flowi4_tos = tos;
	fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1920
	fl4.flowi4_flags = 0;
1921 1922
	fl4.daddr = daddr;
	fl4.saddr = saddr;
1923
	fl4.flowi4_uid = sock_net_uid(net, NULL);
1924
	err = fib_lookup(net, &fl4, res, 0);
1925 1926 1927
	if (err != 0) {
		if (!IN_DEV_FORWARD(in_dev))
			err = -EHOSTUNREACH;
L
Linus Torvalds 已提交
1928
		goto no_route;
1929
	}
L
Linus Torvalds 已提交
1930

1931
	if (res->type == RTN_BROADCAST)
L
Linus Torvalds 已提交
1932 1933
		goto brd_input;

1934
	if (res->type == RTN_LOCAL) {
1935
		err = fib_validate_source(skb, saddr, daddr, tos,
1936
					  0, dev, in_dev, &itag);
1937
		if (err < 0)
1938
			goto martian_source;
L
Linus Torvalds 已提交
1939 1940 1941
		goto local_input;
	}

1942 1943
	if (!IN_DEV_FORWARD(in_dev)) {
		err = -EHOSTUNREACH;
1944
		goto no_route;
1945
	}
1946
	if (res->type != RTN_UNICAST)
L
Linus Torvalds 已提交
1947 1948
		goto martian_destination;

1949
	err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos);
L
Linus Torvalds 已提交
1950 1951 1952 1953 1954 1955
out:	return err;

brd_input:
	if (skb->protocol != htons(ETH_P_IP))
		goto e_inval;

1956
	if (!ipv4_is_zeronet(saddr)) {
1957 1958
		err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
					  in_dev, &itag);
L
Linus Torvalds 已提交
1959
		if (err < 0)
1960
			goto martian_source;
L
Linus Torvalds 已提交
1961 1962
	}
	flags |= RTCF_BROADCAST;
1963
	res->type = RTN_BROADCAST;
L
Linus Torvalds 已提交
1964 1965 1966
	RT_CACHE_STAT_INC(in_brd);

local_input:
1967
	do_cache = false;
1968
	if (res->fi) {
1969
		if (!itag) {
1970
			rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1971
			if (rt_cache_valid(rth)) {
1972 1973 1974
				skb_dst_set_noref(skb, &rth->dst);
				err = 0;
				goto out;
1975 1976 1977 1978 1979
			}
			do_cache = true;
		}
	}

1980
	rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
1981
			   flags | RTCF_LOCAL, res->type,
1982
			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
L
Linus Torvalds 已提交
1983 1984 1985
	if (!rth)
		goto e_nobufs;

1986
	rth->dst.output= ip_rt_bug;
1987 1988 1989
#ifdef CONFIG_IP_ROUTE_CLASSID
	rth->dst.tclassid = itag;
#endif
1990
	rth->rt_is_input = 1;
1991 1992
	if (res->table)
		rth->rt_table_id = res->table->tb_id;
1993

D
Duan Jiong 已提交
1994
	RT_CACHE_STAT_INC(in_slow_tot);
1995
	if (res->type == RTN_UNREACHABLE) {
1996 1997
		rth->dst.input= ip_error;
		rth->dst.error= -err;
L
Linus Torvalds 已提交
1998 1999
		rth->rt_flags 	&= ~RTCF_LOCAL;
	}
2000

2001
	if (do_cache) {
2002
		struct fib_nh *nh = &FIB_RES_NH(*res);
2003 2004 2005 2006 2007 2008 2009 2010 2011

		rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
		if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
			WARN_ON(rth->dst.input == lwtunnel_input);
			rth->dst.lwtstate->orig_input = rth->dst.input;
			rth->dst.input = lwtunnel_input;
		}

		if (unlikely(!rt_cache_route(nh, rth))) {
2012 2013 2014 2015
			rth->dst.flags |= DST_NOCACHE;
			rt_add_uncached_list(rth);
		}
	}
D
David S. Miller 已提交
2016
	skb_dst_set(skb, &rth->dst);
2017
	err = 0;
E
Eric Dumazet 已提交
2018
	goto out;
L
Linus Torvalds 已提交
2019 2020 2021

no_route:
	RT_CACHE_STAT_INC(in_no_route);
2022 2023 2024
	res->type = RTN_UNREACHABLE;
	res->fi = NULL;
	res->table = NULL;
L
Linus Torvalds 已提交
2025 2026 2027 2028 2029 2030 2031 2032
	goto local_input;

	/*
	 *	Do not cache martian addresses: they should be logged (RFC1812)
	 */
martian_destination:
	RT_CACHE_STAT_INC(in_martian_dst);
#ifdef CONFIG_IP_ROUTE_VERBOSE
2033 2034 2035
	if (IN_DEV_LOG_MARTIANS(in_dev))
		net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
				     &daddr, &saddr, dev->name);
L
Linus Torvalds 已提交
2036
#endif
2037

L
Linus Torvalds 已提交
2038 2039
e_inval:
	err = -EINVAL;
E
Eric Dumazet 已提交
2040
	goto out;
L
Linus Torvalds 已提交
2041 2042 2043

e_nobufs:
	err = -ENOBUFS;
E
Eric Dumazet 已提交
2044
	goto out;
L
Linus Torvalds 已提交
2045 2046 2047

martian_source:
	ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
E
Eric Dumazet 已提交
2048
	goto out;
L
Linus Torvalds 已提交
2049 2050
}

2051 2052
int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
			 u8 tos, struct net_device *dev)
L
Linus Torvalds 已提交
2053
{
2054 2055
	struct fib_result res;
	int err;
L
Linus Torvalds 已提交
2056

2057
	tos &= IPTOS_RT_MASK;
2058
	rcu_read_lock();
2059 2060
	err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
	rcu_read_unlock();
2061

2062 2063 2064 2065 2066 2067 2068 2069
	return err;
}
EXPORT_SYMBOL(ip_route_input_noref);

/* called with rcu_read_lock held */
int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
		       u8 tos, struct net_device *dev, struct fib_result *res)
{
L
Linus Torvalds 已提交
2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080
	/* Multicast recognition logic is moved from route cache to here.
	   The problem was that too many Ethernet cards have broken/missing
	   hardware multicast filters :-( As result the host on multicasting
	   network acquires a lot of useless route cache entries, sort of
	   SDR messages from all the world. Now we try to get rid of them.
	   Really, provided software IP multicast filter is organized
	   reasonably (at least, hashed), it does not result in a slowdown
	   comparing with route cache reject entries.
	   Note, that multicast routers are not affected, because
	   route cache entry is created eventually.
	 */
2081
	if (ipv4_is_multicast(daddr)) {
2082
		struct in_device *in_dev = __in_dev_get_rcu(dev);
2083
		int our = 0;
2084
		int err = -EINVAL;
L
Linus Torvalds 已提交
2085

2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100
		if (in_dev)
			our = ip_check_mc_rcu(in_dev, daddr, saddr,
					      ip_hdr(skb)->protocol);

		/* check l3 master if no match yet */
		if ((!in_dev || !our) && netif_is_l3_slave(dev)) {
			struct in_device *l3_in_dev;

			l3_in_dev = __in_dev_get_rcu(skb->dev);
			if (l3_in_dev)
				our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
						      ip_hdr(skb)->protocol);
		}

		if (our
L
Linus Torvalds 已提交
2101
#ifdef CONFIG_IP_MROUTE
2102 2103 2104
			||
		    (!ipv4_is_local_multicast(daddr) &&
		     IN_DEV_MFORWARD(in_dev))
L
Linus Torvalds 已提交
2105
#endif
2106
		   ) {
2107
			err = ip_route_input_mc(skb, daddr, saddr,
2108
						tos, dev, our);
L
Linus Torvalds 已提交
2109
		}
2110
		return err;
L
Linus Torvalds 已提交
2111
	}
2112 2113

	return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
L
Linus Torvalds 已提交
2114 2115
}

E
Eric Dumazet 已提交
2116
/* called with rcu_read_lock() */
2117
static struct rtable *__mkroute_output(const struct fib_result *res,
2118
				       const struct flowi4 *fl4, int orig_oif,
2119
				       struct net_device *dev_out,
2120
				       unsigned int flags)
L
Linus Torvalds 已提交
2121
{
2122
	struct fib_info *fi = res->fi;
2123
	struct fib_nh_exception *fnhe;
2124
	struct in_device *in_dev;
2125
	u16 type = res->type;
2126
	struct rtable *rth;
J
Julian Anastasov 已提交
2127
	bool do_cache;
L
Linus Torvalds 已提交
2128

2129 2130
	in_dev = __in_dev_get_rcu(dev_out);
	if (!in_dev)
2131
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
2132

2133
	if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2134 2135 2136
		if (ipv4_is_loopback(fl4->saddr) &&
		    !(dev_out->flags & IFF_LOOPBACK) &&
		    !netif_is_l3_master(dev_out))
2137 2138
			return ERR_PTR(-EINVAL);

2139
	if (ipv4_is_lbcast(fl4->daddr))
2140
		type = RTN_BROADCAST;
2141
	else if (ipv4_is_multicast(fl4->daddr))
2142
		type = RTN_MULTICAST;
2143
	else if (ipv4_is_zeronet(fl4->daddr))
2144
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
2145 2146 2147 2148

	if (dev_out->flags & IFF_LOOPBACK)
		flags |= RTCF_LOCAL;

2149
	do_cache = true;
2150
	if (type == RTN_BROADCAST) {
L
Linus Torvalds 已提交
2151
		flags |= RTCF_BROADCAST | RTCF_LOCAL;
2152 2153
		fi = NULL;
	} else if (type == RTN_MULTICAST) {
E
Eric Dumazet 已提交
2154
		flags |= RTCF_MULTICAST | RTCF_LOCAL;
2155 2156
		if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
				     fl4->flowi4_proto))
L
Linus Torvalds 已提交
2157
			flags &= ~RTCF_LOCAL;
2158 2159
		else
			do_cache = false;
L
Linus Torvalds 已提交
2160
		/* If multicast route do not exist use
E
Eric Dumazet 已提交
2161 2162
		 * default one, but do not gateway in this case.
		 * Yes, it is hack.
L
Linus Torvalds 已提交
2163
		 */
2164 2165
		if (fi && res->prefixlen < 4)
			fi = NULL;
2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
	} else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
		   (orig_oif != dev_out->ifindex)) {
		/* For local routes that require a particular output interface
		 * we do not want to cache the result.  Caching the result
		 * causes incorrect behaviour when there are multiple source
		 * addresses on the interface, the end result being that if the
		 * intended recipient is waiting on that interface for the
		 * packet he won't receive it because it will be delivered on
		 * the loopback interface and the IP_PKTINFO ipi_ifindex will
		 * be set to the loopback interface as well.
		 */
		fi = NULL;
L
Linus Torvalds 已提交
2178 2179
	}

2180
	fnhe = NULL;
2181 2182
	do_cache &= fi != NULL;
	if (do_cache) {
2183
		struct rtable __rcu **prth;
J
Julian Anastasov 已提交
2184
		struct fib_nh *nh = &FIB_RES_NH(*res);
E
Eric Dumazet 已提交
2185

J
Julian Anastasov 已提交
2186
		fnhe = find_exception(nh, fl4->daddr);
2187
		if (fnhe) {
2188
			prth = &fnhe->fnhe_rth_output;
2189 2190 2191 2192 2193 2194 2195
			rth = rcu_dereference(*prth);
			if (rth && rth->dst.expires &&
			    time_after(jiffies, rth->dst.expires)) {
				ip_del_fnhe(nh, fl4->daddr);
				fnhe = NULL;
			} else {
				goto rt_cache;
J
Julian Anastasov 已提交
2196 2197
			}
		}
2198 2199 2200 2201 2202 2203 2204 2205 2206

		if (unlikely(fl4->flowi4_flags &
			     FLOWI_FLAG_KNOWN_NH &&
			     !(nh->nh_gw &&
			       nh->nh_scope == RT_SCOPE_LINK))) {
			do_cache = false;
			goto add;
		}
		prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2207
		rth = rcu_dereference(*prth);
2208 2209

rt_cache:
2210 2211 2212
		if (rt_cache_valid(rth)) {
			dst_hold(&rth->dst);
			return rth;
2213 2214
		}
	}
J
Julian Anastasov 已提交
2215 2216

add:
2217
	rth = rt_dst_alloc(dev_out, flags, type,
2218
			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
2219
			   IN_DEV_CONF_GET(in_dev, NOXFRM),
J
Julian Anastasov 已提交
2220
			   do_cache);
2221
	if (!rth)
2222
		return ERR_PTR(-ENOBUFS);
2223

2224
	rth->rt_iif	= orig_oif ? : 0;
D
David Ahern 已提交
2225 2226 2227
	if (res->table)
		rth->rt_table_id = res->table->tb_id;

L
Linus Torvalds 已提交
2228 2229 2230
	RT_CACHE_STAT_INC(out_slow_tot);

	if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2231
		if (flags & RTCF_LOCAL &&
L
Linus Torvalds 已提交
2232
		    !(dev_out->flags & IFF_LOOPBACK)) {
2233
			rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2234 2235 2236
			RT_CACHE_STAT_INC(out_slow_mc);
		}
#ifdef CONFIG_IP_MROUTE
2237
		if (type == RTN_MULTICAST) {
L
Linus Torvalds 已提交
2238
			if (IN_DEV_MFORWARD(in_dev) &&
2239
			    !ipv4_is_local_multicast(fl4->daddr)) {
2240 2241
				rth->dst.input = ip_mr_input;
				rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2242 2243 2244 2245 2246
			}
		}
#endif
	}

2247
	rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
2248
	set_lwt_redirect(rth);
L
Linus Torvalds 已提交
2249

2250
	return rth;
L
Linus Torvalds 已提交
2251 2252 2253 2254 2255 2256
}

/*
 * Major route resolver routine.
 */

2257 2258
struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
					const struct sk_buff *skb)
L
Linus Torvalds 已提交
2259
{
2260
	__u8 tos = RT_FL_TOS(fl4);
2261
	struct fib_result res;
2262
	struct rtable *rth;
L
Linus Torvalds 已提交
2263

2264
	res.tclassid	= 0;
L
Linus Torvalds 已提交
2265
	res.fi		= NULL;
2266
	res.table	= NULL;
L
Linus Torvalds 已提交
2267

2268
	fl4->flowi4_iif = LOOPBACK_IFINDEX;
2269 2270 2271
	fl4->flowi4_tos = tos & IPTOS_RT_MASK;
	fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
			 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2272

2273
	rcu_read_lock();
2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290
	rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
	rcu_read_unlock();

	return rth;
}
EXPORT_SYMBOL_GPL(ip_route_output_key_hash);

struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
					    struct fib_result *res,
					    const struct sk_buff *skb)
{
	struct net_device *dev_out = NULL;
	int orig_oif = fl4->flowi4_oif;
	unsigned int flags = 0;
	struct rtable *rth;
	int err = -ENETUNREACH;

2291
	if (fl4->saddr) {
2292
		rth = ERR_PTR(-EINVAL);
2293 2294 2295
		if (ipv4_is_multicast(fl4->saddr) ||
		    ipv4_is_lbcast(fl4->saddr) ||
		    ipv4_is_zeronet(fl4->saddr))
L
Linus Torvalds 已提交
2296 2297 2298 2299
			goto out;

		/* I removed check for oif == dev_out->oif here.
		   It was wrong for two reasons:
2300 2301
		   1. ip_dev_find(net, saddr) can return wrong iface, if saddr
		      is assigned to multiple interfaces.
L
Linus Torvalds 已提交
2302 2303 2304 2305
		   2. Moreover, we are allowed to send packets with saddr
		      of another iface. --ANK
		 */

2306 2307 2308
		if (fl4->flowi4_oif == 0 &&
		    (ipv4_is_multicast(fl4->daddr) ||
		     ipv4_is_lbcast(fl4->daddr))) {
2309
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2310
			dev_out = __ip_dev_find(net, fl4->saddr, false);
2311
			if (!dev_out)
2312 2313
				goto out;

L
Linus Torvalds 已提交
2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328
			/* Special hack: user can direct multicasts
			   and limited broadcast via necessary interface
			   without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
			   This hack is not just for fun, it allows
			   vic,vat and friends to work.
			   They bind socket to loopback, set ttl to zero
			   and expect that it will work.
			   From the viewpoint of routing cache they are broken,
			   because we are not allowed to build multicast path
			   with loopback source addr (look, routing cache
			   cannot know, that ttl is zero, so that packet
			   will not leave this host and route is valid).
			   Luckily, this hack is good workaround.
			 */

2329
			fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2330 2331
			goto make_route;
		}
2332

2333
		if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2334
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2335
			if (!__ip_dev_find(net, fl4->saddr, false))
2336 2337
				goto out;
		}
L
Linus Torvalds 已提交
2338 2339 2340
	}


2341 2342
	if (fl4->flowi4_oif) {
		dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2343
		rth = ERR_PTR(-ENODEV);
2344
		if (!dev_out)
L
Linus Torvalds 已提交
2345
			goto out;
2346 2347

		/* RACE: Check return value of inet_select_addr instead. */
2348
		if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2349
			rth = ERR_PTR(-ENETUNREACH);
2350 2351
			goto out;
		}
2352
		if (ipv4_is_local_multicast(fl4->daddr) ||
2353 2354
		    ipv4_is_lbcast(fl4->daddr) ||
		    fl4->flowi4_proto == IPPROTO_IGMP) {
2355 2356 2357
			if (!fl4->saddr)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
L
Linus Torvalds 已提交
2358 2359
			goto make_route;
		}
2360
		if (!fl4->saddr) {
2361 2362 2363 2364 2365 2366
			if (ipv4_is_multicast(fl4->daddr))
				fl4->saddr = inet_select_addr(dev_out, 0,
							      fl4->flowi4_scope);
			else if (!fl4->daddr)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_HOST);
L
Linus Torvalds 已提交
2367 2368 2369
		}
	}

2370 2371 2372 2373
	if (!fl4->daddr) {
		fl4->daddr = fl4->saddr;
		if (!fl4->daddr)
			fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2374
		dev_out = net->loopback_dev;
2375
		fl4->flowi4_oif = LOOPBACK_IFINDEX;
2376
		res->type = RTN_LOCAL;
L
Linus Torvalds 已提交
2377 2378 2379 2380
		flags |= RTCF_LOCAL;
		goto make_route;
	}

2381
	err = fib_lookup(net, fl4, res, 0);
2382
	if (err) {
2383 2384
		res->fi = NULL;
		res->table = NULL;
2385
		if (fl4->flowi4_oif &&
2386 2387
		    (ipv4_is_multicast(fl4->daddr) ||
		    !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
L
Linus Torvalds 已提交
2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405
			/* Apparently, routing tables are wrong. Assume,
			   that the destination is on link.

			   WHY? DW.
			   Because we are allowed to send to iface
			   even if it has NO routes and NO assigned
			   addresses. When oif is specified, routing
			   tables are looked up with only one purpose:
			   to catch if destination is gatewayed, rather than
			   direct. Moreover, if MSG_DONTROUTE is set,
			   we send packet, ignoring both routing tables
			   and ifaddr state. --ANK


			   We could make it even if oif is unknown,
			   likely IPv6, but we do not.
			 */

2406 2407 2408
			if (fl4->saddr == 0)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
2409
			res->type = RTN_UNICAST;
L
Linus Torvalds 已提交
2410 2411
			goto make_route;
		}
2412
		rth = ERR_PTR(err);
L
Linus Torvalds 已提交
2413 2414 2415
		goto out;
	}

2416
	if (res->type == RTN_LOCAL) {
2417
		if (!fl4->saddr) {
2418 2419
			if (res->fi->fib_prefsrc)
				fl4->saddr = res->fi->fib_prefsrc;
2420
			else
2421
				fl4->saddr = fl4->daddr;
2422
		}
2423 2424

		/* L3 master device is the loopback for that domain */
2425
		dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2426
			net->loopback_dev;
2427
		fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2428 2429 2430 2431
		flags |= RTCF_LOCAL;
		goto make_route;
	}

2432
	fib_select_path(net, res, fl4, skb);
L
Linus Torvalds 已提交
2433

2434
	dev_out = FIB_RES_DEV(*res);
2435
	fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2436 2437 2438


make_route:
2439
	rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
L
Linus Torvalds 已提交
2440

2441
out:
2442
	return rth;
L
Linus Torvalds 已提交
2443
}
2444

2445 2446 2447 2448 2449
static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
{
	return NULL;
}

2450
static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2451
{
2452 2453 2454
	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);

	return mtu ? : dst->dev->mtu;
2455 2456
}

2457 2458
static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
					  struct sk_buff *skb, u32 mtu)
2459 2460 2461
{
}

2462 2463
static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
				       struct sk_buff *skb)
2464 2465 2466
{
}

2467 2468 2469 2470 2471 2472
static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
					  unsigned long old)
{
	return NULL;
}

2473 2474
static struct dst_ops ipv4_dst_blackhole_ops = {
	.family			=	AF_INET,
2475
	.check			=	ipv4_blackhole_dst_check,
2476
	.mtu			=	ipv4_blackhole_mtu,
2477
	.default_advmss		=	ipv4_default_advmss,
2478
	.update_pmtu		=	ipv4_rt_blackhole_update_pmtu,
2479
	.redirect		=	ipv4_rt_blackhole_redirect,
2480
	.cow_metrics		=	ipv4_rt_blackhole_cow_metrics,
2481
	.neigh_lookup		=	ipv4_neigh_lookup,
2482 2483
};

2484
struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2485
{
2486
	struct rtable *ort = (struct rtable *) dst_orig;
2487
	struct rtable *rt;
2488

2489
	rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2490
	if (rt) {
2491
		struct dst_entry *new = &rt->dst;
2492 2493

		new->__use = 1;
2494
		new->input = dst_discard;
E
Eric W. Biederman 已提交
2495
		new->output = dst_discard_out;
2496

2497
		new->dev = ort->dst.dev;
2498 2499 2500
		if (new->dev)
			dev_hold(new->dev);

2501
		rt->rt_is_input = ort->rt_is_input;
2502
		rt->rt_iif = ort->rt_iif;
2503
		rt->rt_pmtu = ort->rt_pmtu;
2504

F
fan.du 已提交
2505
		rt->rt_genid = rt_genid_ipv4(net);
2506 2507 2508
		rt->rt_flags = ort->rt_flags;
		rt->rt_type = ort->rt_type;
		rt->rt_gateway = ort->rt_gateway;
J
Julian Anastasov 已提交
2509
		rt->rt_uses_gateway = ort->rt_uses_gateway;
2510

2511
		INIT_LIST_HEAD(&rt->rt_uncached);
2512 2513 2514
		dst_free(new);
	}

2515 2516 2517
	dst_release(dst_orig);

	return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2518 2519
}

2520
struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2521
				    const struct sock *sk)
L
Linus Torvalds 已提交
2522
{
2523
	struct rtable *rt = __ip_route_output_key(net, flp4);
L
Linus Torvalds 已提交
2524

2525 2526
	if (IS_ERR(rt))
		return rt;
L
Linus Torvalds 已提交
2527

2528
	if (flp4->flowi4_proto)
2529 2530 2531
		rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
							flowi4_to_flowi(flp4),
							sk, 0);
L
Linus Torvalds 已提交
2532

2533
	return rt;
L
Linus Torvalds 已提交
2534
}
2535 2536
EXPORT_SYMBOL_GPL(ip_route_output_flow);

2537
static int rt_fill_info(struct net *net,  __be32 dst, __be32 src, u32 table_id,
2538
			struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2539
			u32 seq)
L
Linus Torvalds 已提交
2540
{
E
Eric Dumazet 已提交
2541
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
2542
	struct rtmsg *r;
2543
	struct nlmsghdr *nlh;
2544
	unsigned long expires = 0;
2545
	u32 error;
J
Julian Anastasov 已提交
2546
	u32 metrics[RTAX_MAX];
2547

2548
	nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2549
	if (!nlh)
2550
		return -EMSGSIZE;
2551 2552

	r = nlmsg_data(nlh);
L
Linus Torvalds 已提交
2553 2554 2555
	r->rtm_family	 = AF_INET;
	r->rtm_dst_len	= 32;
	r->rtm_src_len	= 0;
2556
	r->rtm_tos	= fl4->flowi4_tos;
2557
	r->rtm_table	= table_id < 256 ? table_id : RT_TABLE_COMPAT;
2558
	if (nla_put_u32(skb, RTA_TABLE, table_id))
D
David S. Miller 已提交
2559
		goto nla_put_failure;
L
Linus Torvalds 已提交
2560 2561 2562 2563 2564 2565
	r->rtm_type	= rt->rt_type;
	r->rtm_scope	= RT_SCOPE_UNIVERSE;
	r->rtm_protocol = RTPROT_UNSPEC;
	r->rtm_flags	= (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
	if (rt->rt_flags & RTCF_NOTIFY)
		r->rtm_flags |= RTM_F_NOTIFY;
2566 2567
	if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
		r->rtm_flags |= RTCF_DOREDIRECT;
2568

2569
	if (nla_put_in_addr(skb, RTA_DST, dst))
D
David S. Miller 已提交
2570
		goto nla_put_failure;
2571
	if (src) {
L
Linus Torvalds 已提交
2572
		r->rtm_src_len = 32;
2573
		if (nla_put_in_addr(skb, RTA_SRC, src))
D
David S. Miller 已提交
2574
			goto nla_put_failure;
L
Linus Torvalds 已提交
2575
	}
D
David S. Miller 已提交
2576 2577 2578
	if (rt->dst.dev &&
	    nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
		goto nla_put_failure;
2579
#ifdef CONFIG_IP_ROUTE_CLASSID
D
David S. Miller 已提交
2580 2581 2582
	if (rt->dst.tclassid &&
	    nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
		goto nla_put_failure;
L
Linus Torvalds 已提交
2583
#endif
2584
	if (!rt_is_input_route(rt) &&
2585
	    fl4->saddr != src) {
2586
		if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
D
David S. Miller 已提交
2587 2588
			goto nla_put_failure;
	}
J
Julian Anastasov 已提交
2589
	if (rt->rt_uses_gateway &&
2590
	    nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
D
David S. Miller 已提交
2591
		goto nla_put_failure;
2592

2593 2594 2595 2596 2597 2598 2599 2600 2601 2602
	expires = rt->dst.expires;
	if (expires) {
		unsigned long now = jiffies;

		if (time_before(now, expires))
			expires -= now;
		else
			expires = 0;
	}

J
Julian Anastasov 已提交
2603
	memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2604
	if (rt->rt_pmtu && expires)
J
Julian Anastasov 已提交
2605 2606
		metrics[RTAX_MTU - 1] = rt->rt_pmtu;
	if (rtnetlink_put_metrics(skb, metrics) < 0)
2607 2608
		goto nla_put_failure;

2609
	if (fl4->flowi4_mark &&
2610
	    nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
D
David S. Miller 已提交
2611
		goto nla_put_failure;
E
Eric Dumazet 已提交
2612

2613 2614 2615 2616 2617
	if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
	    nla_put_u32(skb, RTA_UID,
			from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
		goto nla_put_failure;

2618
	error = rt->dst.error;
2619

2620
	if (rt_is_input_route(rt)) {
2621 2622 2623 2624 2625
#ifdef CONFIG_IP_MROUTE
		if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
		    IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
			int err = ipmr_get_route(net, skb,
						 fl4->saddr, fl4->daddr,
2626
						 r, portid);
2627

2628
			if (err <= 0) {
D
David Ahern 已提交
2629 2630 2631
				if (err == 0)
					return 0;
				goto nla_put_failure;
2632 2633 2634
			}
		} else
#endif
2635
			if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2636
				goto nla_put_failure;
L
Linus Torvalds 已提交
2637 2638
	}

2639
	if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2640
		goto nla_put_failure;
2641

2642 2643
	nlmsg_end(skb, nlh);
	return 0;
L
Linus Torvalds 已提交
2644

2645
nla_put_failure:
2646 2647
	nlmsg_cancel(skb, nlh);
	return -EMSGSIZE;
L
Linus Torvalds 已提交
2648 2649
}

2650 2651
static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
			     struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
2652
{
2653
	struct net *net = sock_net(in_skb->sk);
2654 2655
	struct rtmsg *rtm;
	struct nlattr *tb[RTA_MAX+1];
L
Linus Torvalds 已提交
2656
	struct rtable *rt = NULL;
2657
	struct flowi4 fl4;
A
Al Viro 已提交
2658 2659 2660
	__be32 dst = 0;
	__be32 src = 0;
	u32 iif;
2661
	int err;
E
Eric Dumazet 已提交
2662
	int mark;
L
Linus Torvalds 已提交
2663
	struct sk_buff *skb;
2664
	u32 table_id = RT_TABLE_MAIN;
2665
	kuid_t uid;
L
Linus Torvalds 已提交
2666

2667
	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy,
2668
			  extack);
2669 2670 2671 2672 2673
	if (err < 0)
		goto errout;

	rtm = nlmsg_data(nlh);

L
Linus Torvalds 已提交
2674
	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2675
	if (!skb) {
2676 2677 2678
		err = -ENOBUFS;
		goto errout;
	}
L
Linus Torvalds 已提交
2679 2680 2681 2682

	/* Reserve room for dummy headers, this skb can pass
	   through good chunk of routing engine.
	 */
2683
	skb_reset_mac_header(skb);
2684
	skb_reset_network_header(skb);
2685

2686 2687
	src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
	dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2688
	iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
E
Eric Dumazet 已提交
2689
	mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2690 2691 2692 2693
	if (tb[RTA_UID])
		uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
	else
		uid = (iif ? INVALID_UID : current_uid());
L
Linus Torvalds 已提交
2694

2695 2696 2697 2698 2699 2700 2701 2702 2703
	/* Bugfix: need to give ip_route_input enough of an IP header to
	 * not gag.
	 */
	ip_hdr(skb)->protocol = IPPROTO_UDP;
	ip_hdr(skb)->saddr = src;
	ip_hdr(skb)->daddr = dst;

	skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));

2704 2705 2706 2707 2708 2709
	memset(&fl4, 0, sizeof(fl4));
	fl4.daddr = dst;
	fl4.saddr = src;
	fl4.flowi4_tos = rtm->rtm_tos;
	fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
	fl4.flowi4_mark = mark;
2710
	fl4.flowi4_uid = uid;
2711

L
Linus Torvalds 已提交
2712
	if (iif) {
2713 2714
		struct net_device *dev;

2715
		dev = __dev_get_by_index(net, iif);
2716
		if (!dev) {
2717 2718 2719 2720
			err = -ENODEV;
			goto errout_free;
		}

L
Linus Torvalds 已提交
2721 2722
		skb->protocol	= htons(ETH_P_IP);
		skb->dev	= dev;
E
Eric Dumazet 已提交
2723
		skb->mark	= mark;
L
Linus Torvalds 已提交
2724
		err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2725

E
Eric Dumazet 已提交
2726
		rt = skb_rtable(skb);
2727 2728
		if (err == 0 && rt->dst.error)
			err = -rt->dst.error;
L
Linus Torvalds 已提交
2729
	} else {
2730
		rt = ip_route_output_key(net, &fl4);
2731 2732 2733 2734

		err = 0;
		if (IS_ERR(rt))
			err = PTR_ERR(rt);
L
Linus Torvalds 已提交
2735
	}
2736

L
Linus Torvalds 已提交
2737
	if (err)
2738
		goto errout_free;
L
Linus Torvalds 已提交
2739

2740
	skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
2741 2742 2743
	if (rtm->rtm_flags & RTM_F_NOTIFY)
		rt->rt_flags |= RTCF_NOTIFY;

2744 2745 2746 2747
	if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
		table_id = rt->rt_table_id;

	err = rt_fill_info(net, dst, src, table_id, &fl4, skb,
2748
			   NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2749
	if (err < 0)
2750
		goto errout_free;
L
Linus Torvalds 已提交
2751

2752
	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2753
errout:
2754
	return err;
L
Linus Torvalds 已提交
2755

2756
errout_free:
L
Linus Torvalds 已提交
2757
	kfree_skb(skb);
2758
	goto errout;
L
Linus Torvalds 已提交
2759 2760 2761 2762
}

void ip_rt_multicast_event(struct in_device *in_dev)
{
2763
	rt_cache_flush(dev_net(in_dev->dev));
L
Linus Torvalds 已提交
2764 2765 2766
}

#ifdef CONFIG_SYSCTL
2767 2768 2769 2770
static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
static int ip_rt_gc_min_interval __read_mostly	= HZ / 2;
static int ip_rt_gc_elasticity __read_mostly	= 8;

2771
static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2772
					void __user *buffer,
L
Linus Torvalds 已提交
2773 2774
					size_t *lenp, loff_t *ppos)
{
2775 2776
	struct net *net = (struct net *)__ctl->extra1;

L
Linus Torvalds 已提交
2777
	if (write) {
2778 2779
		rt_cache_flush(net);
		fnhe_genid_bump(net);
L
Linus Torvalds 已提交
2780
		return 0;
2781
	}
L
Linus Torvalds 已提交
2782 2783 2784 2785

	return -EINVAL;
}

2786
static struct ctl_table ipv4_route_table[] = {
L
Linus Torvalds 已提交
2787 2788 2789 2790 2791
	{
		.procname	= "gc_thresh",
		.data		= &ipv4_dst_ops.gc_thresh,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2792
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2793 2794 2795 2796 2797 2798
	},
	{
		.procname	= "max_size",
		.data		= &ip_rt_max_size,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2799
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2800 2801 2802
	},
	{
		/*  Deprecated. Use gc_min_interval_ms */
2803

L
Linus Torvalds 已提交
2804 2805 2806 2807
		.procname	= "gc_min_interval",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2808
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
2809 2810 2811 2812 2813 2814
	},
	{
		.procname	= "gc_min_interval_ms",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2815
		.proc_handler	= proc_dointvec_ms_jiffies,
L
Linus Torvalds 已提交
2816 2817 2818 2819 2820 2821
	},
	{
		.procname	= "gc_timeout",
		.data		= &ip_rt_gc_timeout,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2822
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
2823
	},
2824 2825 2826 2827 2828 2829 2830
	{
		.procname	= "gc_interval",
		.data		= &ip_rt_gc_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec_jiffies,
	},
L
Linus Torvalds 已提交
2831 2832 2833 2834 2835
	{
		.procname	= "redirect_load",
		.data		= &ip_rt_redirect_load,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2836
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2837 2838 2839 2840 2841 2842
	},
	{
		.procname	= "redirect_number",
		.data		= &ip_rt_redirect_number,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2843
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2844 2845 2846 2847 2848 2849
	},
	{
		.procname	= "redirect_silence",
		.data		= &ip_rt_redirect_silence,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2850
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2851 2852 2853 2854 2855 2856
	},
	{
		.procname	= "error_cost",
		.data		= &ip_rt_error_cost,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2857
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2858 2859 2860 2861 2862 2863
	},
	{
		.procname	= "error_burst",
		.data		= &ip_rt_error_burst,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2864
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2865 2866 2867 2868 2869 2870
	},
	{
		.procname	= "gc_elasticity",
		.data		= &ip_rt_gc_elasticity,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2871
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2872 2873 2874 2875 2876 2877
	},
	{
		.procname	= "mtu_expires",
		.data		= &ip_rt_mtu_expires,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2878
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
2879 2880 2881 2882 2883 2884
	},
	{
		.procname	= "min_pmtu",
		.data		= &ip_rt_min_pmtu,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2885
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2886 2887 2888 2889 2890 2891
	},
	{
		.procname	= "min_adv_mss",
		.data		= &ip_rt_min_advmss,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
2892
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
2893
	},
2894
	{ }
L
Linus Torvalds 已提交
2895
};
2896 2897 2898 2899 2900 2901

static struct ctl_table ipv4_route_flush_table[] = {
	{
		.procname	= "flush",
		.maxlen		= sizeof(int),
		.mode		= 0200,
A
Alexey Dobriyan 已提交
2902
		.proc_handler	= ipv4_sysctl_rtcache_flush,
2903
	},
2904
	{ },
2905 2906 2907 2908 2909 2910 2911
};

static __net_init int sysctl_route_net_init(struct net *net)
{
	struct ctl_table *tbl;

	tbl = ipv4_route_flush_table;
O
Octavian Purdila 已提交
2912
	if (!net_eq(net, &init_net)) {
2913
		tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2914
		if (!tbl)
2915
			goto err_dup;
2916 2917 2918 2919

		/* Don't export sysctls to unprivileged users */
		if (net->user_ns != &init_user_ns)
			tbl[0].procname = NULL;
2920 2921 2922
	}
	tbl[0].extra1 = net;

2923
	net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2924
	if (!net->ipv4.route_hdr)
2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948
		goto err_reg;
	return 0;

err_reg:
	if (tbl != ipv4_route_flush_table)
		kfree(tbl);
err_dup:
	return -ENOMEM;
}

static __net_exit void sysctl_route_net_exit(struct net *net)
{
	struct ctl_table *tbl;

	tbl = net->ipv4.route_hdr->ctl_table_arg;
	unregister_net_sysctl_table(net->ipv4.route_hdr);
	BUG_ON(tbl == ipv4_route_flush_table);
	kfree(tbl);
}

static __net_initdata struct pernet_operations sysctl_route_ops = {
	.init = sysctl_route_net_init,
	.exit = sysctl_route_net_exit,
};
L
Linus Torvalds 已提交
2949 2950
#endif

2951
static __net_init int rt_genid_init(struct net *net)
2952
{
F
fan.du 已提交
2953
	atomic_set(&net->ipv4.rt_genid, 0);
2954
	atomic_set(&net->fnhe_genid, 0);
2955 2956
	get_random_bytes(&net->ipv4.dev_addr_genid,
			 sizeof(net->ipv4.dev_addr_genid));
2957 2958 2959
	return 0;
}

2960 2961
static __net_initdata struct pernet_operations rt_genid_ops = {
	.init = rt_genid_init,
2962 2963
};

2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979
static int __net_init ipv4_inetpeer_init(struct net *net)
{
	struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);

	if (!bp)
		return -ENOMEM;
	inet_peer_base_init(bp);
	net->ipv4.peers = bp;
	return 0;
}

static void __net_exit ipv4_inetpeer_exit(struct net *net)
{
	struct inet_peer_base *bp = net->ipv4.peers;

	net->ipv4.peers = NULL;
2980
	inetpeer_invalidate_tree(bp);
2981 2982 2983 2984 2985 2986 2987
	kfree(bp);
}

static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
	.init	=	ipv4_inetpeer_init,
	.exit	=	ipv4_inetpeer_exit,
};
2988

2989
#ifdef CONFIG_IP_ROUTE_CLASSID
2990
struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2991
#endif /* CONFIG_IP_ROUTE_CLASSID */
L
Linus Torvalds 已提交
2992 2993 2994

int __init ip_rt_init(void)
{
2995
	int rc = 0;
E
Eric Dumazet 已提交
2996
	int cpu;
L
Linus Torvalds 已提交
2997

E
Eric Dumazet 已提交
2998 2999 3000 3001 3002 3003
	ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
	if (!ip_idents)
		panic("IP: failed to allocate ip_idents\n");

	prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));

E
Eric Dumazet 已提交
3004 3005 3006 3007
	ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
	if (!ip_tstamps)
		panic("IP: failed to allocate ip_tstamps\n");

E
Eric Dumazet 已提交
3008 3009 3010 3011 3012 3013
	for_each_possible_cpu(cpu) {
		struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);

		INIT_LIST_HEAD(&ul->head);
		spin_lock_init(&ul->lock);
	}
3014
#ifdef CONFIG_IP_ROUTE_CLASSID
3015
	ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
L
Linus Torvalds 已提交
3016 3017 3018 3019
	if (!ip_rt_acct)
		panic("IP: failed to allocate ip_rt_acct\n");
#endif

A
Alexey Dobriyan 已提交
3020 3021
	ipv4_dst_ops.kmem_cachep =
		kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3022
				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3023

3024 3025
	ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;

3026 3027 3028 3029 3030 3031
	if (dst_entries_init(&ipv4_dst_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_ops counter\n");

	if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");

D
David S. Miller 已提交
3032 3033
	ipv4_dst_ops.gc_thresh = ~0;
	ip_rt_max_size = INT_MAX;
L
Linus Torvalds 已提交
3034 3035 3036 3037

	devinet_init();
	ip_fib_init();

3038
	if (ip_rt_proc_init())
J
Joe Perches 已提交
3039
		pr_err("Unable to create route proc files\n");
L
Linus Torvalds 已提交
3040 3041
#ifdef CONFIG_XFRM
	xfrm_init();
3042
	xfrm4_init();
L
Linus Torvalds 已提交
3043
#endif
3044
	rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
3045

3046 3047 3048
#ifdef CONFIG_SYSCTL
	register_pernet_subsys(&sysctl_route_ops);
#endif
3049
	register_pernet_subsys(&rt_genid_ops);
3050
	register_pernet_subsys(&ipv4_inetpeer_ops);
L
Linus Torvalds 已提交
3051 3052 3053
	return rc;
}

3054
#ifdef CONFIG_SYSCTL
A
Al Viro 已提交
3055 3056 3057 3058 3059 3060
/*
 * We really need to sanitize the damn ipv4 init order, then all
 * this nonsense will go away.
 */
void __init ip_static_sysctl_init(void)
{
3061
	register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
A
Al Viro 已提交
3062
}
3063
#endif