route.c 81.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		ROUTE - implementation of the IP router.
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Linus Torvalds, <Linus.Torvalds@helsinki.fi>
 *		Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 * Fixes:
 *		Alan Cox	:	Verify area fixes.
 *		Alan Cox	:	cli() protects routing changes
 *		Rui Oliveira	:	ICMP routing table updates
 *		(rco@di.uminho.pt)	Routing table insertion and update
 *		Linus Torvalds	:	Rewrote bits to be sensible
 *		Alan Cox	:	Added BSD route gw semantics
21
 *		Alan Cox	:	Super /proc >4K
L
Linus Torvalds 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
 *		Alan Cox	:	MTU in route table
 *		Alan Cox	: 	MSS actually. Also added the window
 *					clamper.
 *		Sam Lantinga	:	Fixed route matching in rt_del()
 *		Alan Cox	:	Routing cache support.
 *		Alan Cox	:	Removed compatibility cruft.
 *		Alan Cox	:	RTF_REJECT support.
 *		Alan Cox	:	TCP irtt support.
 *		Jonathan Naylor	:	Added Metric support.
 *	Miquel van Smoorenburg	:	BSD API fixes.
 *	Miquel van Smoorenburg	:	Metrics.
 *		Alan Cox	:	Use __u32 properly
 *		Alan Cox	:	Aligned routing errors more closely with BSD
 *					our system is still very different.
 *		Alan Cox	:	Faster /proc handling
 *	Alexey Kuznetsov	:	Massive rework to support tree based routing,
 *					routing caches and better behaviour.
39
 *
L
Linus Torvalds 已提交
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
 *		Olaf Erb	:	irtt wasn't being copied right.
 *		Bjorn Ekwall	:	Kerneld route support.
 *		Alan Cox	:	Multicast fixed (I hope)
 * 		Pavel Krauz	:	Limited broadcast fixed
 *		Mike McLagan	:	Routing by source
 *	Alexey Kuznetsov	:	End of old history. Split to fib.c and
 *					route.c and rewritten from scratch.
 *		Andi Kleen	:	Load-limit warning messages.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
 *	Vitaly E. Lavrov	:	Race condition in ip_route_input_slow.
 *	Tobias Ringstrom	:	Uninitialized res.type in ip_route_output_slow.
 *	Vladimir V. Ivanov	:	IP rule info (flowid) is really useful.
 *		Marc Boucher	:	routing by fwmark
 *	Robert Olsson		:	Added rt_cache statistics
 *	Arnaldo C. Melo		:	Convert proc stuff to seq_file
55
 *	Eric Dumazet		:	hashed spinlocks and rt_check_expire() fixes.
56 57
 * 	Ilia Sotnikov		:	Ignore TOS on PMTUD and Redirect
 * 	Ilia Sotnikov		:	Removed TOS from hash calculations
L
Linus Torvalds 已提交
58 59 60 61 62 63 64 65 66 67 68 69 70 71
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 */

#include <linux/module.h>
#include <asm/uaccess.h>
#include <asm/system.h>
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
72
#include <linux/bootmem.h>
L
Linus Torvalds 已提交
73 74 75 76 77 78 79 80 81
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
82
#include <linux/workqueue.h>
L
Linus Torvalds 已提交
83 84 85 86 87 88 89 90 91 92
#include <linux/skbuff.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include <linux/pkt_sched.h>
#include <linux/mroute.h>
#include <linux/netfilter_ipv4.h>
#include <linux/random.h>
#include <linux/jhash.h>
#include <linux/rcupdate.h>
#include <linux/times.h>
93
#include <linux/slab.h>
94
#include <net/dst.h>
95
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104 105
#include <net/protocol.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/inetpeer.h>
#include <net/sock.h>
#include <net/ip_fib.h>
#include <net/arp.h>
#include <net/tcp.h>
#include <net/icmp.h>
#include <net/xfrm.h>
106
#include <net/netevent.h>
107
#include <net/rtnetlink.h>
L
Linus Torvalds 已提交
108 109 110 111
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif

112 113
#define RT_FL_TOS(oldflp4) \
    ((u32)(oldflp4->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))
L
Linus Torvalds 已提交
114 115 116 117 118 119

#define IP_MAX_MTU	0xFFF0

#define RT_GC_TIMEOUT (300*HZ)

static int ip_rt_max_size;
120 121 122 123 124 125 126 127 128 129 130 131
static int ip_rt_gc_timeout __read_mostly	= RT_GC_TIMEOUT;
static int ip_rt_gc_interval __read_mostly	= 60 * HZ;
static int ip_rt_gc_min_interval __read_mostly	= HZ / 2;
static int ip_rt_redirect_number __read_mostly	= 9;
static int ip_rt_redirect_load __read_mostly	= HZ / 50;
static int ip_rt_redirect_silence __read_mostly	= ((HZ / 50) << (9 + 1));
static int ip_rt_error_cost __read_mostly	= HZ;
static int ip_rt_error_burst __read_mostly	= 5 * HZ;
static int ip_rt_gc_elasticity __read_mostly	= 8;
static int ip_rt_mtu_expires __read_mostly	= 10 * 60 * HZ;
static int ip_rt_min_pmtu __read_mostly		= 512 + 20 + 20;
static int ip_rt_min_advmss __read_mostly	= 256;
132
static int rt_chain_length_max __read_mostly	= 20;
L
Linus Torvalds 已提交
133 134 135 136 137 138

/*
 *	Interface to generic destination cache.
 */

static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
139
static unsigned int	 ipv4_default_advmss(const struct dst_entry *dst);
140
static unsigned int	 ipv4_default_mtu(const struct dst_entry *dst);
L
Linus Torvalds 已提交
141 142 143 144
static void		 ipv4_dst_destroy(struct dst_entry *dst);
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
static void		 ipv4_link_failure(struct sk_buff *skb);
static void		 ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
145
static int rt_garbage_collect(struct dst_ops *ops);
L
Linus Torvalds 已提交
146

E
Eric Dumazet 已提交
147 148 149 150
static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
			    int how)
{
}
L
Linus Torvalds 已提交
151

152 153
static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
{
154 155 156 157 158
	struct rtable *rt = (struct rtable *) dst;
	struct inet_peer *peer;
	u32 *p = NULL;

	if (!rt->peer)
159
		rt_bind_peer(rt, rt->rt_dst, 1);
160

161 162
	peer = rt->peer;
	if (peer) {
163 164 165
		u32 *old_p = __DST_METRICS_PTR(old);
		unsigned long prev, new;

166 167 168
		p = peer->metrics;
		if (inet_metrics_new(peer))
			memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186

		new = (unsigned long) p;
		prev = cmpxchg(&dst->_metrics, old, new);

		if (prev != old) {
			p = __DST_METRICS_PTR(prev);
			if (prev & DST_METRICS_READ_ONLY)
				p = NULL;
		} else {
			if (rt->fi) {
				fib_info_put(rt->fi);
				rt->fi = NULL;
			}
		}
	}
	return p;
}

L
Linus Torvalds 已提交
187 188
static struct dst_ops ipv4_dst_ops = {
	.family =		AF_INET,
189
	.protocol =		cpu_to_be16(ETH_P_IP),
L
Linus Torvalds 已提交
190 191
	.gc =			rt_garbage_collect,
	.check =		ipv4_dst_check,
192
	.default_advmss =	ipv4_default_advmss,
193
	.default_mtu =		ipv4_default_mtu,
194
	.cow_metrics =		ipv4_cow_metrics,
L
Linus Torvalds 已提交
195 196 197 198 199
	.destroy =		ipv4_dst_destroy,
	.ifdown =		ipv4_dst_ifdown,
	.negative_advice =	ipv4_negative_advice,
	.link_failure =		ipv4_link_failure,
	.update_pmtu =		ip_rt_update_pmtu,
200
	.local_out =		__ip_local_out,
L
Linus Torvalds 已提交
201 202 203 204
};

#define ECN_OR_COST(class)	TC_PRIO_##class

205
const __u8 ip_tos2prio[16] = {
L
Linus Torvalds 已提交
206
	TC_PRIO_BESTEFFORT,
D
Dan Siemon 已提交
207
	ECN_OR_COST(BESTEFFORT),
L
Linus Torvalds 已提交
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
	TC_PRIO_BESTEFFORT,
	ECN_OR_COST(BESTEFFORT),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK)
};


/*
 * Route cache.
 */

/* The locking scheme is rather straight forward:
 *
 * 1) Read-Copy Update protects the buckets of the central route hash.
 * 2) Only writers remove entries, and they hold the lock
 *    as they look at rtable reference counts.
 * 3) Only readers acquire references to rtable entries,
 *    they do so with atomic increments and with the
 *    lock held.
 */

struct rt_hash_bucket {
240
	struct rtable __rcu	*chain;
241
};
242

243 244
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
	defined(CONFIG_PROVE_LOCKING)
245 246 247
/*
 * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
 * The size of this table is a power of two and depends on the number of CPUS.
248
 * (on lockdep we have a quite big spinlock_t, so keep the size down there)
249
 */
250 251
#ifdef CONFIG_LOCKDEP
# define RT_HASH_LOCK_SZ	256
252
#else
253 254 255 256 257 258 259 260 261 262 263
# if NR_CPUS >= 32
#  define RT_HASH_LOCK_SZ	4096
# elif NR_CPUS >= 16
#  define RT_HASH_LOCK_SZ	2048
# elif NR_CPUS >= 8
#  define RT_HASH_LOCK_SZ	1024
# elif NR_CPUS >= 4
#  define RT_HASH_LOCK_SZ	512
# else
#  define RT_HASH_LOCK_SZ	256
# endif
264 265 266 267
#endif

static spinlock_t	*rt_hash_locks;
# define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
268 269 270 271 272 273 274 275 276 277 278 279 280

static __init void rt_hash_lock_init(void)
{
	int i;

	rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
			GFP_KERNEL);
	if (!rt_hash_locks)
		panic("IP: failed to allocate rt_hash_locks\n");

	for (i = 0; i < RT_HASH_LOCK_SZ; i++)
		spin_lock_init(&rt_hash_locks[i]);
}
281 282
#else
# define rt_hash_lock_addr(slot) NULL
283 284 285 286

static inline void rt_hash_lock_init(void)
{
}
287
#endif
L
Linus Torvalds 已提交
288

289 290 291
static struct rt_hash_bucket 	*rt_hash_table __read_mostly;
static unsigned			rt_hash_mask __read_mostly;
static unsigned int		rt_hash_log  __read_mostly;
L
Linus Torvalds 已提交
292

293
static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
294
#define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
L
Linus Torvalds 已提交
295

296
static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
297
				   int genid)
L
Linus Torvalds 已提交
298
{
299
	return jhash_3words((__force u32)daddr, (__force u32)saddr,
300
			    idx, genid)
301
		& rt_hash_mask;
L
Linus Torvalds 已提交
302 303
}

304 305 306 307 308
static inline int rt_genid(struct net *net)
{
	return atomic_read(&net->ipv4.rt_genid);
}

L
Linus Torvalds 已提交
309 310
#ifdef CONFIG_PROC_FS
struct rt_cache_iter_state {
311
	struct seq_net_private p;
L
Linus Torvalds 已提交
312
	int bucket;
313
	int genid;
L
Linus Torvalds 已提交
314 315
};

316
static struct rtable *rt_cache_get_first(struct seq_file *seq)
L
Linus Torvalds 已提交
317
{
318
	struct rt_cache_iter_state *st = seq->private;
L
Linus Torvalds 已提交
319 320 321
	struct rtable *r = NULL;

	for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
322
		if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain))
323
			continue;
L
Linus Torvalds 已提交
324
		rcu_read_lock_bh();
325
		r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
326
		while (r) {
327
			if (dev_net(r->dst.dev) == seq_file_net(seq) &&
328
			    r->rt_genid == st->genid)
329
				return r;
330
			r = rcu_dereference_bh(r->dst.rt_next);
331
		}
L
Linus Torvalds 已提交
332 333
		rcu_read_unlock_bh();
	}
334
	return r;
L
Linus Torvalds 已提交
335 336
}

337
static struct rtable *__rt_cache_get_next(struct seq_file *seq,
338
					  struct rtable *r)
L
Linus Torvalds 已提交
339
{
340
	struct rt_cache_iter_state *st = seq->private;
341

342
	r = rcu_dereference_bh(r->dst.rt_next);
L
Linus Torvalds 已提交
343 344
	while (!r) {
		rcu_read_unlock_bh();
345 346 347
		do {
			if (--st->bucket < 0)
				return NULL;
348
		} while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain));
L
Linus Torvalds 已提交
349
		rcu_read_lock_bh();
350
		r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
L
Linus Torvalds 已提交
351
	}
352
	return r;
L
Linus Torvalds 已提交
353 354
}

355
static struct rtable *rt_cache_get_next(struct seq_file *seq,
356 357
					struct rtable *r)
{
358 359
	struct rt_cache_iter_state *st = seq->private;
	while ((r = __rt_cache_get_next(seq, r)) != NULL) {
360
		if (dev_net(r->dst.dev) != seq_file_net(seq))
361
			continue;
362 363 364 365 366 367
		if (r->rt_genid == st->genid)
			break;
	}
	return r;
}

368
static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
L
Linus Torvalds 已提交
369
{
370
	struct rtable *r = rt_cache_get_first(seq);
L
Linus Torvalds 已提交
371 372

	if (r)
373
		while (pos && (r = rt_cache_get_next(seq, r)))
L
Linus Torvalds 已提交
374 375 376 377 378 379
			--pos;
	return pos ? NULL : r;
}

static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
{
380 381
	struct rt_cache_iter_state *st = seq->private;
	if (*pos)
382
		return rt_cache_get_idx(seq, *pos - 1);
383
	st->genid = rt_genid(seq_file_net(seq));
384
	return SEQ_START_TOKEN;
L
Linus Torvalds 已提交
385 386 387 388
}

static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
389
	struct rtable *r;
L
Linus Torvalds 已提交
390 391

	if (v == SEQ_START_TOKEN)
392
		r = rt_cache_get_first(seq);
L
Linus Torvalds 已提交
393
	else
394
		r = rt_cache_get_next(seq, v);
L
Linus Torvalds 已提交
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
	++*pos;
	return r;
}

static void rt_cache_seq_stop(struct seq_file *seq, void *v)
{
	if (v && v != SEQ_START_TOKEN)
		rcu_read_unlock_bh();
}

static int rt_cache_seq_show(struct seq_file *seq, void *v)
{
	if (v == SEQ_START_TOKEN)
		seq_printf(seq, "%-127s\n",
			   "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
			   "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
			   "HHUptod\tSpecDst");
	else {
		struct rtable *r = v;
414
		int len;
L
Linus Torvalds 已提交
415

416 417
		seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
			      "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
418
			r->dst.dev ? r->dst.dev->name : "*",
419 420
			(__force u32)r->rt_dst,
			(__force u32)r->rt_gateway,
421 422
			r->rt_flags, atomic_read(&r->dst.__refcnt),
			r->dst.__use, 0, (__force u32)r->rt_src,
423
			dst_metric_advmss(&r->dst) + 40,
424 425 426
			dst_metric(&r->dst, RTAX_WINDOW),
			(int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
			      dst_metric(&r->dst, RTAX_RTTVAR)),
427
			r->rt_key_tos,
428 429
			r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
			r->dst.hh ? (r->dst.hh->hh_output ==
L
Linus Torvalds 已提交
430
				       dev_queue_xmit) : 0,
431 432 433
			r->rt_spec_dst, &len);

		seq_printf(seq, "%*s\n", 127 - len, "");
434 435
	}
	return 0;
L
Linus Torvalds 已提交
436 437
}

438
static const struct seq_operations rt_cache_seq_ops = {
L
Linus Torvalds 已提交
439 440 441 442 443 444 445 446
	.start  = rt_cache_seq_start,
	.next   = rt_cache_seq_next,
	.stop   = rt_cache_seq_stop,
	.show   = rt_cache_seq_show,
};

static int rt_cache_seq_open(struct inode *inode, struct file *file)
{
447
	return seq_open_net(inode, file, &rt_cache_seq_ops,
448
			sizeof(struct rt_cache_iter_state));
L
Linus Torvalds 已提交
449 450
}

451
static const struct file_operations rt_cache_seq_fops = {
L
Linus Torvalds 已提交
452 453 454 455
	.owner	 = THIS_MODULE,
	.open	 = rt_cache_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
456
	.release = seq_release_net,
L
Linus Torvalds 已提交
457 458 459 460 461 462 463 464 465 466
};


static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
{
	int cpu;

	if (*pos == 0)
		return SEQ_START_TOKEN;

467
	for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
468 469 470
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
471
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
472 473 474 475 476 477 478 479
	}
	return NULL;
}

static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	int cpu;

480
	for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
481 482 483
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
484
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
485 486
	}
	return NULL;
487

L
Linus Torvalds 已提交
488 489 490 491 492 493 494 495 496 497 498 499
}

static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
{

}

static int rt_cpu_seq_show(struct seq_file *seq, void *v)
{
	struct rt_cache_stat *st = v;

	if (v == SEQ_START_TOKEN) {
500
		seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
L
Linus Torvalds 已提交
501 502
		return 0;
	}
503

L
Linus Torvalds 已提交
504 505
	seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
		   " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
506
		   dst_entries_get_slow(&ipv4_dst_ops),
L
Linus Torvalds 已提交
507 508 509 510 511 512 513 514 515 516
		   st->in_hit,
		   st->in_slow_tot,
		   st->in_slow_mc,
		   st->in_no_route,
		   st->in_brd,
		   st->in_martian_dst,
		   st->in_martian_src,

		   st->out_hit,
		   st->out_slow_tot,
517
		   st->out_slow_mc,
L
Linus Torvalds 已提交
518 519 520 521 522 523 524 525 526 527 528

		   st->gc_total,
		   st->gc_ignored,
		   st->gc_goal_miss,
		   st->gc_dst_overflow,
		   st->in_hlist_search,
		   st->out_hlist_search
		);
	return 0;
}

529
static const struct seq_operations rt_cpu_seq_ops = {
L
Linus Torvalds 已提交
530 531 532 533 534 535 536 537 538 539 540 541
	.start  = rt_cpu_seq_start,
	.next   = rt_cpu_seq_next,
	.stop   = rt_cpu_seq_stop,
	.show   = rt_cpu_seq_show,
};


static int rt_cpu_seq_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &rt_cpu_seq_ops);
}

542
static const struct file_operations rt_cpu_seq_fops = {
L
Linus Torvalds 已提交
543 544 545 546 547 548 549
	.owner	 = THIS_MODULE,
	.open	 = rt_cpu_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
	.release = seq_release,
};

550
#ifdef CONFIG_IP_ROUTE_CLASSID
551
static int rt_acct_proc_show(struct seq_file *m, void *v)
552
{
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567
	struct ip_rt_acct *dst, *src;
	unsigned int i, j;

	dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
	if (!dst)
		return -ENOMEM;

	for_each_possible_cpu(i) {
		src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
		for (j = 0; j < 256; j++) {
			dst[j].o_bytes   += src[j].o_bytes;
			dst[j].o_packets += src[j].o_packets;
			dst[j].i_bytes   += src[j].i_bytes;
			dst[j].i_packets += src[j].i_packets;
		}
568 569
	}

570 571 572 573
	seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
	kfree(dst);
	return 0;
}
574

575 576 577
static int rt_acct_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, rt_acct_proc_show, NULL);
578
}
579 580 581 582 583 584 585 586

static const struct file_operations rt_acct_proc_fops = {
	.owner		= THIS_MODULE,
	.open		= rt_acct_proc_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};
587
#endif
588

589
static int __net_init ip_rt_do_proc_init(struct net *net)
590 591 592 593 594 595 596 597
{
	struct proc_dir_entry *pde;

	pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
			&rt_cache_seq_fops);
	if (!pde)
		goto err1;

598 599
	pde = proc_create("rt_cache", S_IRUGO,
			  net->proc_net_stat, &rt_cpu_seq_fops);
600 601 602
	if (!pde)
		goto err2;

603
#ifdef CONFIG_IP_ROUTE_CLASSID
604
	pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
605 606 607 608 609
	if (!pde)
		goto err3;
#endif
	return 0;

610
#ifdef CONFIG_IP_ROUTE_CLASSID
611 612 613 614 615 616 617 618
err3:
	remove_proc_entry("rt_cache", net->proc_net_stat);
#endif
err2:
	remove_proc_entry("rt_cache", net->proc_net);
err1:
	return -ENOMEM;
}
619 620 621 622 623

static void __net_exit ip_rt_do_proc_exit(struct net *net)
{
	remove_proc_entry("rt_cache", net->proc_net_stat);
	remove_proc_entry("rt_cache", net->proc_net);
624
#ifdef CONFIG_IP_ROUTE_CLASSID
625
	remove_proc_entry("rt_acct", net->proc_net);
626
#endif
627 628 629 630 631 632 633 634 635 636 637 638
}

static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
	.init = ip_rt_do_proc_init,
	.exit = ip_rt_do_proc_exit,
};

static int __init ip_rt_proc_init(void)
{
	return register_pernet_subsys(&ip_rt_proc_ops);
}

639
#else
640
static inline int ip_rt_proc_init(void)
641 642 643
{
	return 0;
}
L
Linus Torvalds 已提交
644
#endif /* CONFIG_PROC_FS */
645

S
Stephen Hemminger 已提交
646
static inline void rt_free(struct rtable *rt)
L
Linus Torvalds 已提交
647
{
648
	call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
L
Linus Torvalds 已提交
649 650
}

S
Stephen Hemminger 已提交
651
static inline void rt_drop(struct rtable *rt)
L
Linus Torvalds 已提交
652 653
{
	ip_rt_put(rt);
654
	call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
L
Linus Torvalds 已提交
655 656
}

S
Stephen Hemminger 已提交
657
static inline int rt_fast_clean(struct rtable *rth)
L
Linus Torvalds 已提交
658 659 660 661
{
	/* Kill broadcast/multicast entries very aggresively, if they
	   collide in hash table with more useful entries */
	return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
662
		rt_is_input_route(rth) && rth->dst.rt_next;
L
Linus Torvalds 已提交
663 664
}

S
Stephen Hemminger 已提交
665
static inline int rt_valuable(struct rtable *rth)
L
Linus Torvalds 已提交
666 667
{
	return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
668
		(rth->peer && rth->peer->pmtu_expires);
L
Linus Torvalds 已提交
669 670 671 672 673 674 675
}

static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
{
	unsigned long age;
	int ret = 0;

676
	if (atomic_read(&rth->dst.__refcnt))
L
Linus Torvalds 已提交
677 678
		goto out;

679
	age = jiffies - rth->dst.lastuse;
L
Linus Torvalds 已提交
680 681 682 683 684 685 686 687 688 689 690 691 692 693
	if ((age <= tmo1 && !rt_fast_clean(rth)) ||
	    (age <= tmo2 && rt_valuable(rth)))
		goto out;
	ret = 1;
out:	return ret;
}

/* Bits of score are:
 * 31: very valuable
 * 30: not quite useless
 * 29..0: usage counter
 */
static inline u32 rt_score(struct rtable *rt)
{
694
	u32 score = jiffies - rt->dst.lastuse;
L
Linus Torvalds 已提交
695 696 697 698 699 700

	score = ~score & ~(3<<30);

	if (rt_valuable(rt))
		score |= (1<<31);

701
	if (rt_is_output_route(rt) ||
L
Linus Torvalds 已提交
702 703 704 705 706 707
	    !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
		score |= (1<<30);

	return score;
}

708 709 710 711 712 713
static inline bool rt_caching(const struct net *net)
{
	return net->ipv4.current_rt_cache_rebuild_count <=
		net->ipv4.sysctl_rt_cache_rebuild_count;
}

714 715
static inline bool compare_hash_inputs(const struct rtable *rt1,
				       const struct rtable *rt2)
716
{
717 718 719
	return ((((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
		((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
		(rt1->rt_iif ^ rt2->rt_iif)) == 0);
720 721
}

722
static inline int compare_keys(struct rtable *rt1, struct rtable *rt2)
L
Linus Torvalds 已提交
723
{
724 725 726
	return (((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
		((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
		(rt1->rt_mark ^ rt2->rt_mark) |
727
		(rt1->rt_key_tos ^ rt2->rt_key_tos) |
728 729
		(rt1->rt_oif ^ rt2->rt_oif) |
		(rt1->rt_iif ^ rt2->rt_iif)) == 0;
L
Linus Torvalds 已提交
730 731
}

732 733
static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
{
734
	return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
735 736
}

737 738
static inline int rt_is_expired(struct rtable *rth)
{
739
	return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
740 741
}

742 743 744 745 746
/*
 * Perform a full scan of hash table and free all entries.
 * Can be called by a softirq or a process.
 * In the later case, we want to be reschedule if necessary
 */
747
static void rt_do_flush(struct net *net, int process_context)
748 749 750 751 752
{
	unsigned int i;
	struct rtable *rth, *next;

	for (i = 0; i <= rt_hash_mask; i++) {
753 754 755
		struct rtable __rcu **pprev;
		struct rtable *list;

756 757
		if (process_context && need_resched())
			cond_resched();
758
		rth = rcu_dereference_raw(rt_hash_table[i].chain);
759 760 761 762
		if (!rth)
			continue;

		spin_lock_bh(rt_hash_lock_addr(i));
763

764 765 766
		list = NULL;
		pprev = &rt_hash_table[i].chain;
		rth = rcu_dereference_protected(*pprev,
767
			lockdep_is_held(rt_hash_lock_addr(i)));
768

769 770
		while (rth) {
			next = rcu_dereference_protected(rth->dst.rt_next,
771
				lockdep_is_held(rt_hash_lock_addr(i)));
772 773 774 775 776 777

			if (!net ||
			    net_eq(dev_net(rth->dst.dev), net)) {
				rcu_assign_pointer(*pprev, next);
				rcu_assign_pointer(rth->dst.rt_next, list);
				list = rth;
778
			} else {
779
				pprev = &rth->dst.rt_next;
780
			}
781
			rth = next;
782
		}
783

784 785
		spin_unlock_bh(rt_hash_lock_addr(i));

786 787 788
		for (; list; list = next) {
			next = rcu_dereference_protected(list->dst.rt_next, 1);
			rt_free(list);
789 790 791 792
		}
	}
}

793 794 795 796 797 798 799 800 801 802 803
/*
 * While freeing expired entries, we compute average chain length
 * and standard deviation, using fixed-point arithmetic.
 * This to have an estimation of rt_chain_length_max
 *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
 * We use 3 bits for frational part, and 29 (or 61) for magnitude.
 */

#define FRACT_BITS 3
#define ONE (1UL << FRACT_BITS)

E
Eric Dumazet 已提交
804 805 806 807 808 809 810 811 812 813 814 815
/*
 * Given a hash chain and an item in this hash chain,
 * find if a previous entry has the same hash_inputs
 * (but differs on tos, mark or oif)
 * Returns 0 if an alias is found.
 * Returns ONE if rth has no alias before itself.
 */
static int has_noalias(const struct rtable *head, const struct rtable *rth)
{
	const struct rtable *aux = head;

	while (aux != rth) {
816
		if (compare_hash_inputs(aux, rth))
E
Eric Dumazet 已提交
817
			return 0;
818
		aux = rcu_dereference_protected(aux->dst.rt_next, 1);
E
Eric Dumazet 已提交
819 820 821 822
	}
	return ONE;
}

823
/*
L
Lucas De Marchi 已提交
824
 * Perturbation of rt_genid by a small quantity [1..256]
825 826 827
 * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
 * many times (2^24) without giving recent rt_genid.
 * Jenkins hash is strong enough that litle changes of rt_genid are OK.
L
Linus Torvalds 已提交
828
 */
829
static void rt_cache_invalidate(struct net *net)
L
Linus Torvalds 已提交
830
{
831
	unsigned char shuffle;
L
Linus Torvalds 已提交
832

833
	get_random_bytes(&shuffle, sizeof(shuffle));
834
	atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
L
Linus Torvalds 已提交
835 836
}

837 838 839 840
/*
 * delay < 0  : invalidate cache (fast : entries will be deleted later)
 * delay >= 0 : invalidate & flush cache (can be long)
 */
841
void rt_cache_flush(struct net *net, int delay)
L
Linus Torvalds 已提交
842
{
843
	rt_cache_invalidate(net);
844
	if (delay >= 0)
845
		rt_do_flush(net, !in_softirq());
L
Linus Torvalds 已提交
846 847
}

848
/* Flush previous cache invalidated entries from the cache */
849
void rt_cache_flush_batch(struct net *net)
850
{
851
	rt_do_flush(net, !in_softirq());
852 853
}

854 855
static void rt_emergency_hash_rebuild(struct net *net)
{
856
	if (net_ratelimit())
857
		printk(KERN_WARNING "Route hash chain too long!\n");
858
	rt_cache_invalidate(net);
859 860
}

L
Linus Torvalds 已提交
861 862 863 864 865 866 867 868 869 870 871 872 873
/*
   Short description of GC goals.

   We want to build algorithm, which will keep routing cache
   at some equilibrium point, when number of aged off entries
   is kept approximately equal to newly generated ones.

   Current expiration strength is variable "expire".
   We try to adjust it dynamically, so that if networking
   is idle expires is large enough to keep enough of warm entries,
   and when load increases it reduces to limit cache size.
 */

874
static int rt_garbage_collect(struct dst_ops *ops)
L
Linus Torvalds 已提交
875 876 877 878 879
{
	static unsigned long expire = RT_GC_TIMEOUT;
	static unsigned long last_gc;
	static int rover;
	static int equilibrium;
880 881
	struct rtable *rth;
	struct rtable __rcu **rthp;
L
Linus Torvalds 已提交
882 883
	unsigned long now = jiffies;
	int goal;
884
	int entries = dst_entries_get_fast(&ipv4_dst_ops);
L
Linus Torvalds 已提交
885 886 887 888 889 890 891 892 893

	/*
	 * Garbage collection is pretty expensive,
	 * do not make it too frequently.
	 */

	RT_CACHE_STAT_INC(gc_total);

	if (now - last_gc < ip_rt_gc_min_interval &&
894
	    entries < ip_rt_max_size) {
L
Linus Torvalds 已提交
895 896 897 898
		RT_CACHE_STAT_INC(gc_ignored);
		goto out;
	}

899
	entries = dst_entries_get_slow(&ipv4_dst_ops);
L
Linus Torvalds 已提交
900
	/* Calculate number of entries, which we want to expire now. */
901
	goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
L
Linus Torvalds 已提交
902 903 904
	if (goal <= 0) {
		if (equilibrium < ipv4_dst_ops.gc_thresh)
			equilibrium = ipv4_dst_ops.gc_thresh;
905
		goal = entries - equilibrium;
L
Linus Torvalds 已提交
906
		if (goal > 0) {
907
			equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
908
			goal = entries - equilibrium;
L
Linus Torvalds 已提交
909 910 911 912 913
		}
	} else {
		/* We are in dangerous area. Try to reduce cache really
		 * aggressively.
		 */
914
		goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
915
		equilibrium = entries - goal;
L
Linus Torvalds 已提交
916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933
	}

	if (now - last_gc >= ip_rt_gc_min_interval)
		last_gc = now;

	if (goal <= 0) {
		equilibrium += goal;
		goto work_done;
	}

	do {
		int i, k;

		for (i = rt_hash_mask, k = rover; i >= 0; i--) {
			unsigned long tmo = expire;

			k = (k + 1) & rt_hash_mask;
			rthp = &rt_hash_table[k].chain;
934
			spin_lock_bh(rt_hash_lock_addr(k));
935 936
			while ((rth = rcu_dereference_protected(*rthp,
					lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
937
				if (!rt_is_expired(rth) &&
938
					!rt_may_expire(rth, tmo, expire)) {
L
Linus Torvalds 已提交
939
					tmo >>= 1;
940
					rthp = &rth->dst.rt_next;
L
Linus Torvalds 已提交
941 942
					continue;
				}
943
				*rthp = rth->dst.rt_next;
L
Linus Torvalds 已提交
944 945 946
				rt_free(rth);
				goal--;
			}
947
			spin_unlock_bh(rt_hash_lock_addr(k));
L
Linus Torvalds 已提交
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971
			if (goal <= 0)
				break;
		}
		rover = k;

		if (goal <= 0)
			goto work_done;

		/* Goal is not achieved. We stop process if:

		   - if expire reduced to zero. Otherwise, expire is halfed.
		   - if table is not full.
		   - if we are called from interrupt.
		   - jiffies check is just fallback/debug loop breaker.
		     We will not spin here for long time in any case.
		 */

		RT_CACHE_STAT_INC(gc_goal_miss);

		if (expire == 0)
			break;

		expire >>= 1;

972
		if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
L
Linus Torvalds 已提交
973 974 975
			goto out;
	} while (!in_softirq() && time_before_eq(jiffies, now));

976 977 978
	if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
		goto out;
	if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
L
Linus Torvalds 已提交
979 980 981 982 983 984 985 986 987
		goto out;
	if (net_ratelimit())
		printk(KERN_WARNING "dst cache overflow\n");
	RT_CACHE_STAT_INC(gc_dst_overflow);
	return 1;

work_done:
	expire += ip_rt_gc_min_interval;
	if (expire > ip_rt_gc_timeout ||
988 989
	    dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
	    dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
L
Linus Torvalds 已提交
990 991 992 993
		expire = ip_rt_gc_timeout;
out:	return 0;
}

E
Eric Dumazet 已提交
994 995 996 997 998 999 1000 1001 1002 1003
/*
 * Returns number of entries in a hash chain that have different hash_inputs
 */
static int slow_chain_length(const struct rtable *head)
{
	int length = 0;
	const struct rtable *rth = head;

	while (rth) {
		length += has_noalias(head, rth);
1004
		rth = rcu_dereference_protected(rth->dst.rt_next, 1);
E
Eric Dumazet 已提交
1005 1006 1007 1008
	}
	return length >> FRACT_BITS;
}

1009 1010
static struct rtable *rt_intern_hash(unsigned hash, struct rtable *rt,
				     struct sk_buff *skb, int ifindex)
L
Linus Torvalds 已提交
1011
{
1012 1013
	struct rtable	*rth, *cand;
	struct rtable __rcu **rthp, **candp;
L
Linus Torvalds 已提交
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
	unsigned long	now;
	u32 		min_score;
	int		chain_length;
	int attempts = !in_softirq();

restart:
	chain_length = 0;
	min_score = ~(u32)0;
	cand = NULL;
	candp = NULL;
	now = jiffies;

1026
	if (!rt_caching(dev_net(rt->dst.dev))) {
1027 1028 1029 1030 1031 1032 1033 1034
		/*
		 * If we're not caching, just tell the caller we
		 * were successful and don't touch the route.  The
		 * caller hold the sole reference to the cache entry, and
		 * it will be released when the caller is done with it.
		 * If we drop it here, the callers have no way to resolve routes
		 * when we're not caching.  Instead, just point *rp at rt, so
		 * the caller gets a single use out of the route
1035 1036 1037
		 * Note that we do rt_free on this new route entry, so that
		 * once its refcount hits zero, we are still able to reap it
		 * (Thanks Alexey)
E
Eric Dumazet 已提交
1038 1039 1040
		 * Note: To avoid expensive rcu stuff for this uncached dst,
		 * we set DST_NOCACHE so that dst_release() can free dst without
		 * waiting a grace period.
1041
		 */
1042

E
Eric Dumazet 已提交
1043
		rt->dst.flags |= DST_NOCACHE;
1044
		if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1045
			int err = arp_bind_neighbour(&rt->dst);
1046 1047 1048 1049
			if (err) {
				if (net_ratelimit())
					printk(KERN_WARNING
					    "Neighbour table failure & not caching routes.\n");
E
Eric Dumazet 已提交
1050
				ip_rt_put(rt);
1051
				return ERR_PTR(err);
1052 1053 1054 1055
			}
		}

		goto skip_hashing;
1056 1057
	}

L
Linus Torvalds 已提交
1058 1059
	rthp = &rt_hash_table[hash].chain;

1060
	spin_lock_bh(rt_hash_lock_addr(hash));
1061 1062
	while ((rth = rcu_dereference_protected(*rthp,
			lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1063
		if (rt_is_expired(rth)) {
1064
			*rthp = rth->dst.rt_next;
1065 1066 1067
			rt_free(rth);
			continue;
		}
1068
		if (compare_keys(rth, rt) && compare_netns(rth, rt)) {
L
Linus Torvalds 已提交
1069
			/* Put it first */
1070
			*rthp = rth->dst.rt_next;
L
Linus Torvalds 已提交
1071 1072 1073 1074 1075
			/*
			 * Since lookup is lockfree, the deletion
			 * must be visible to another weakly ordered CPU before
			 * the insertion at the start of the hash chain.
			 */
1076
			rcu_assign_pointer(rth->dst.rt_next,
L
Linus Torvalds 已提交
1077 1078 1079 1080 1081 1082 1083
					   rt_hash_table[hash].chain);
			/*
			 * Since lookup is lockfree, the update writes
			 * must be ordered for consistency on SMP.
			 */
			rcu_assign_pointer(rt_hash_table[hash].chain, rth);

1084
			dst_use(&rth->dst, now);
1085
			spin_unlock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1086 1087

			rt_drop(rt);
1088
			if (skb)
1089
				skb_dst_set(skb, &rth->dst);
1090
			return rth;
L
Linus Torvalds 已提交
1091 1092
		}

1093
		if (!atomic_read(&rth->dst.__refcnt)) {
L
Linus Torvalds 已提交
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
			u32 score = rt_score(rth);

			if (score <= min_score) {
				cand = rth;
				candp = rthp;
				min_score = score;
			}
		}

		chain_length++;

1105
		rthp = &rth->dst.rt_next;
L
Linus Torvalds 已提交
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115
	}

	if (cand) {
		/* ip_rt_gc_elasticity used to be average length of chain
		 * length, when exceeded gc becomes really aggressive.
		 *
		 * The second limit is less certain. At the moment it allows
		 * only 2 entries per bucket. We will see.
		 */
		if (chain_length > ip_rt_gc_elasticity) {
1116
			*candp = cand->dst.rt_next;
L
Linus Torvalds 已提交
1117 1118
			rt_free(cand);
		}
1119
	} else {
E
Eric Dumazet 已提交
1120 1121
		if (chain_length > rt_chain_length_max &&
		    slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1122
			struct net *net = dev_net(rt->dst.dev);
1123
			int num = ++net->ipv4.current_rt_cache_rebuild_count;
1124
			if (!rt_caching(net)) {
1125
				printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
1126
					rt->dst.dev->name, num);
1127
			}
1128
			rt_emergency_hash_rebuild(net);
1129 1130
			spin_unlock_bh(rt_hash_lock_addr(hash));

1131
			hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1132 1133
					ifindex, rt_genid(net));
			goto restart;
1134
		}
L
Linus Torvalds 已提交
1135 1136 1137 1138 1139
	}

	/* Try to bind route to arp only if it is output
	   route or unicast forwarding path.
	 */
1140
	if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1141
		int err = arp_bind_neighbour(&rt->dst);
L
Linus Torvalds 已提交
1142
		if (err) {
1143
			spin_unlock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1144 1145 1146

			if (err != -ENOBUFS) {
				rt_drop(rt);
1147
				return ERR_PTR(err);
L
Linus Torvalds 已提交
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158
			}

			/* Neighbour tables are full and nothing
			   can be released. Try to shrink route cache,
			   it is most likely it holds some neighbour records.
			 */
			if (attempts-- > 0) {
				int saved_elasticity = ip_rt_gc_elasticity;
				int saved_int = ip_rt_gc_min_interval;
				ip_rt_gc_elasticity	= 1;
				ip_rt_gc_min_interval	= 0;
1159
				rt_garbage_collect(&ipv4_dst_ops);
L
Linus Torvalds 已提交
1160 1161 1162 1163 1164 1165
				ip_rt_gc_min_interval	= saved_int;
				ip_rt_gc_elasticity	= saved_elasticity;
				goto restart;
			}

			if (net_ratelimit())
1166
				printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
L
Linus Torvalds 已提交
1167
			rt_drop(rt);
1168
			return ERR_PTR(-ENOBUFS);
L
Linus Torvalds 已提交
1169 1170 1171
		}
	}

1172
	rt->dst.rt_next = rt_hash_table[hash].chain;
1173

1174 1175
	/*
	 * Since lookup is lockfree, we must make sure
L
Lucas De Marchi 已提交
1176
	 * previous writes to rt are committed to memory
1177 1178
	 * before making rt visible to other CPUS.
	 */
1179
	rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1180

1181
	spin_unlock_bh(rt_hash_lock_addr(hash));
1182

1183
skip_hashing:
1184
	if (skb)
1185
		skb_dst_set(skb, &rt->dst);
1186
	return rt;
L
Linus Torvalds 已提交
1187 1188
}

1189 1190 1191 1192 1193 1194 1195
static atomic_t __rt_peer_genid = ATOMIC_INIT(0);

static u32 rt_peer_genid(void)
{
	return atomic_read(&__rt_peer_genid);
}

1196
void rt_bind_peer(struct rtable *rt, __be32 daddr, int create)
L
Linus Torvalds 已提交
1197 1198 1199
{
	struct inet_peer *peer;

1200
	peer = inet_getpeer_v4(daddr, create);
L
Linus Torvalds 已提交
1201

1202
	if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
L
Linus Torvalds 已提交
1203
		inet_putpeer(peer);
1204 1205
	else
		rt->rt_peer_genid = rt_peer_genid();
L
Linus Torvalds 已提交
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
}

/*
 * Peer allocation may fail only in serious out-of-memory conditions.  However
 * we still can generate some output.
 * Random ID selection looks a bit dangerous because we have no chances to
 * select ID being unique in a reasonable period of time.
 * But broken packet identifier may be better than no packet at all.
 */
static void ip_select_fb_ident(struct iphdr *iph)
{
	static DEFINE_SPINLOCK(ip_fb_id_lock);
	static u32 ip_fallback_id;
	u32 salt;

	spin_lock_bh(&ip_fb_id_lock);
1222
	salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
L
Linus Torvalds 已提交
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
	iph->id = htons(salt & 0xFFFF);
	ip_fallback_id = salt;
	spin_unlock_bh(&ip_fb_id_lock);
}

void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
{
	struct rtable *rt = (struct rtable *) dst;

	if (rt) {
		if (rt->peer == NULL)
1234
			rt_bind_peer(rt, rt->rt_dst, 1);
L
Linus Torvalds 已提交
1235 1236 1237 1238 1239 1240 1241 1242 1243

		/* If peer is attached to destination, it is never detached,
		   so that we need not to grab a lock to dereference it.
		 */
		if (rt->peer) {
			iph->id = htons(inet_getid(rt->peer, more));
			return;
		}
	} else
1244
		printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
1245
		       __builtin_return_address(0));
L
Linus Torvalds 已提交
1246 1247 1248

	ip_select_fb_ident(iph);
}
E
Eric Dumazet 已提交
1249
EXPORT_SYMBOL(__ip_select_ident);
L
Linus Torvalds 已提交
1250 1251 1252

static void rt_del(unsigned hash, struct rtable *rt)
{
1253 1254
	struct rtable __rcu **rthp;
	struct rtable *aux;
L
Linus Torvalds 已提交
1255

1256
	rthp = &rt_hash_table[hash].chain;
1257
	spin_lock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1258
	ip_rt_put(rt);
1259 1260
	while ((aux = rcu_dereference_protected(*rthp,
			lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1261
		if (aux == rt || rt_is_expired(aux)) {
1262
			*rthp = aux->dst.rt_next;
1263 1264
			rt_free(aux);
			continue;
L
Linus Torvalds 已提交
1265
		}
1266
		rthp = &aux->dst.rt_next;
1267
	}
1268
	spin_unlock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1269 1270
}

1271
/* called in rcu_read_lock() section */
A
Al Viro 已提交
1272 1273
void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
		    __be32 saddr, struct net_device *dev)
L
Linus Torvalds 已提交
1274
{
1275
	struct in_device *in_dev = __in_dev_get_rcu(dev);
1276
	struct inet_peer *peer;
1277
	struct net *net;
L
Linus Torvalds 已提交
1278 1279 1280 1281

	if (!in_dev)
		return;

1282
	net = dev_net(dev);
1283 1284 1285
	if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
	    ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
	    ipv4_is_zeronet(new_gw))
L
Linus Torvalds 已提交
1286 1287 1288 1289 1290 1291 1292 1293
		goto reject_redirect;

	if (!IN_DEV_SHARED_MEDIA(in_dev)) {
		if (!inet_addr_onlink(in_dev, new_gw, old_gw))
			goto reject_redirect;
		if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
			goto reject_redirect;
	} else {
1294
		if (inet_addr_type(net, new_gw) != RTN_UNICAST)
L
Linus Torvalds 已提交
1295 1296 1297
			goto reject_redirect;
	}

1298 1299 1300
	peer = inet_getpeer_v4(daddr, 1);
	if (peer) {
		peer->redirect_learned.a4 = new_gw;
1301

1302
		inet_putpeer(peer);
L
Linus Torvalds 已提交
1303

1304
		atomic_inc(&__rt_peer_genid);
L
Linus Torvalds 已提交
1305 1306 1307 1308 1309 1310
	}
	return;

reject_redirect:
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
1311 1312 1313 1314
		printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
			"  Advised path = %pI4 -> %pI4\n",
		       &old_gw, dev->name, &new_gw,
		       &saddr, &daddr);
L
Linus Torvalds 已提交
1315
#endif
1316
	;
L
Linus Torvalds 已提交
1317 1318 1319 1320
}

static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
{
1321
	struct rtable *rt = (struct rtable *)dst;
L
Linus Torvalds 已提交
1322 1323 1324
	struct dst_entry *ret = dst;

	if (rt) {
T
Timo Teräs 已提交
1325
		if (dst->obsolete > 0) {
L
Linus Torvalds 已提交
1326 1327
			ip_rt_put(rt);
			ret = NULL;
1328
		} else if (rt->rt_flags & RTCF_REDIRECTED) {
1329 1330
			unsigned hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
						rt->rt_oif,
1331
						rt_genid(dev_net(dst->dev)));
L
Linus Torvalds 已提交
1332 1333
			rt_del(hash, rt);
			ret = NULL;
1334 1335 1336 1337 1338 1339 1340 1341
		} else if (rt->peer &&
			   rt->peer->pmtu_expires &&
			   time_after_eq(jiffies, rt->peer->pmtu_expires)) {
			unsigned long orig = rt->peer->pmtu_expires;

			if (cmpxchg(&rt->peer->pmtu_expires, orig, 0) == orig)
				dst_metric_set(dst, RTAX_MTU,
					       rt->peer->pmtu_orig);
L
Linus Torvalds 已提交
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
		}
	}
	return ret;
}

/*
 * Algorithm:
 *	1. The first ip_rt_redirect_number redirects are sent
 *	   with exponential backoff, then we stop sending them at all,
 *	   assuming that the host ignores our redirects.
 *	2. If we did not see packets requiring redirects
 *	   during ip_rt_redirect_silence, we assume that the host
 *	   forgot redirected route and start to send redirects again.
 *
 * This algorithm is much cheaper and more intelligent than dumb load limiting
 * in icmp.c.
 *
 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 * and "frag. need" (breaks PMTU discovery) in icmp.c.
 */

void ip_rt_send_redirect(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
1365
	struct rtable *rt = skb_rtable(skb);
1366
	struct in_device *in_dev;
1367
	struct inet_peer *peer;
1368
	int log_martians;
L
Linus Torvalds 已提交
1369

1370
	rcu_read_lock();
1371
	in_dev = __in_dev_get_rcu(rt->dst.dev);
1372 1373
	if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
		rcu_read_unlock();
L
Linus Torvalds 已提交
1374
		return;
1375 1376 1377
	}
	log_martians = IN_DEV_LOG_MARTIANS(in_dev);
	rcu_read_unlock();
L
Linus Torvalds 已提交
1378

1379
	if (!rt->peer)
1380
		rt_bind_peer(rt, rt->rt_dst, 1);
1381 1382 1383 1384 1385 1386
	peer = rt->peer;
	if (!peer) {
		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
		return;
	}

L
Linus Torvalds 已提交
1387 1388 1389
	/* No redirected packets during ip_rt_redirect_silence;
	 * reset the algorithm.
	 */
1390 1391
	if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
		peer->rate_tokens = 0;
L
Linus Torvalds 已提交
1392 1393

	/* Too many ignored redirects; do not send anything
1394
	 * set dst.rate_last to the last seen redirected packet.
L
Linus Torvalds 已提交
1395
	 */
1396 1397
	if (peer->rate_tokens >= ip_rt_redirect_number) {
		peer->rate_last = jiffies;
1398
		return;
L
Linus Torvalds 已提交
1399 1400 1401 1402 1403
	}

	/* Check for load limit; set rate_last to the latest sent
	 * redirect.
	 */
1404
	if (peer->rate_tokens == 0 ||
1405
	    time_after(jiffies,
1406 1407
		       (peer->rate_last +
			(ip_rt_redirect_load << peer->rate_tokens)))) {
L
Linus Torvalds 已提交
1408
		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1409 1410
		peer->rate_last = jiffies;
		++peer->rate_tokens;
L
Linus Torvalds 已提交
1411
#ifdef CONFIG_IP_ROUTE_VERBOSE
1412
		if (log_martians &&
1413
		    peer->rate_tokens == ip_rt_redirect_number &&
L
Linus Torvalds 已提交
1414
		    net_ratelimit())
1415
			printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
1416
			       &ip_hdr(skb)->saddr, rt->rt_iif,
1417
				&rt->rt_dst, &rt->rt_gateway);
L
Linus Torvalds 已提交
1418 1419 1420 1421 1422 1423
#endif
	}
}

static int ip_error(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
1424
	struct rtable *rt = skb_rtable(skb);
1425
	struct inet_peer *peer;
L
Linus Torvalds 已提交
1426
	unsigned long now;
1427
	bool send;
L
Linus Torvalds 已提交
1428 1429
	int code;

1430
	switch (rt->dst.error) {
L
Linus Torvalds 已提交
1431 1432 1433 1434 1435 1436 1437 1438
		case EINVAL:
		default:
			goto out;
		case EHOSTUNREACH:
			code = ICMP_HOST_UNREACH;
			break;
		case ENETUNREACH:
			code = ICMP_NET_UNREACH;
1439
			IP_INC_STATS_BH(dev_net(rt->dst.dev),
1440
					IPSTATS_MIB_INNOROUTES);
L
Linus Torvalds 已提交
1441 1442 1443 1444 1445 1446
			break;
		case EACCES:
			code = ICMP_PKT_FILTERED;
			break;
	}

1447
	if (!rt->peer)
1448
		rt_bind_peer(rt, rt->rt_dst, 1);
1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
	peer = rt->peer;

	send = true;
	if (peer) {
		now = jiffies;
		peer->rate_tokens += now - peer->rate_last;
		if (peer->rate_tokens > ip_rt_error_burst)
			peer->rate_tokens = ip_rt_error_burst;
		peer->rate_last = now;
		if (peer->rate_tokens >= ip_rt_error_cost)
			peer->rate_tokens -= ip_rt_error_cost;
		else
			send = false;
L
Linus Torvalds 已提交
1462
	}
1463 1464
	if (send)
		icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
L
Linus Torvalds 已提交
1465 1466 1467

out:	kfree_skb(skb);
	return 0;
1468
}
L
Linus Torvalds 已提交
1469 1470 1471 1472 1473 1474

/*
 *	The last two values are not from the RFC but
 *	are needed for AMPRnet AX.25 paths.
 */

1475
static const unsigned short mtu_plateau[] =
L
Linus Torvalds 已提交
1476 1477
{32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };

S
Stephen Hemminger 已提交
1478
static inline unsigned short guess_mtu(unsigned short old_mtu)
L
Linus Torvalds 已提交
1479 1480
{
	int i;
1481

L
Linus Torvalds 已提交
1482 1483 1484 1485 1486 1487
	for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
		if (old_mtu > mtu_plateau[i])
			return mtu_plateau[i];
	return 68;
}

1488
unsigned short ip_rt_frag_needed(struct net *net, const struct iphdr *iph,
1489 1490
				 unsigned short new_mtu,
				 struct net_device *dev)
L
Linus Torvalds 已提交
1491 1492 1493
{
	unsigned short old_mtu = ntohs(iph->tot_len);
	unsigned short est_mtu = 0;
1494
	struct inet_peer *peer;
L
Linus Torvalds 已提交
1495

1496 1497 1498
	peer = inet_getpeer_v4(iph->daddr, 1);
	if (peer) {
		unsigned short mtu = new_mtu;
L
Linus Torvalds 已提交
1499

1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
		if (new_mtu < 68 || new_mtu >= old_mtu) {
			/* BSD 4.2 derived systems incorrectly adjust
			 * tot_len by the IP header length, and report
			 * a zero MTU in the ICMP message.
			 */
			if (mtu == 0 &&
			    old_mtu >= 68 + (iph->ihl << 2))
				old_mtu -= iph->ihl << 2;
			mtu = guess_mtu(old_mtu);
		}
1510

1511 1512 1513
		if (mtu < ip_rt_min_pmtu)
			mtu = ip_rt_min_pmtu;
		if (!peer->pmtu_expires || mtu < peer->pmtu_learned) {
H
Hiroaki SHIMODA 已提交
1514 1515 1516 1517 1518 1519
			unsigned long pmtu_expires;

			pmtu_expires = jiffies + ip_rt_mtu_expires;
			if (!pmtu_expires)
				pmtu_expires = 1UL;

1520 1521
			est_mtu = mtu;
			peer->pmtu_learned = mtu;
H
Hiroaki SHIMODA 已提交
1522
			peer->pmtu_expires = pmtu_expires;
1523
		}
L
Linus Torvalds 已提交
1524

1525
		inet_putpeer(peer);
L
Linus Torvalds 已提交
1526

1527
		atomic_inc(&__rt_peer_genid);
L
Linus Torvalds 已提交
1528 1529 1530 1531
	}
	return est_mtu ? : new_mtu;
}

1532 1533 1534 1535
static void check_peer_pmtu(struct dst_entry *dst, struct inet_peer *peer)
{
	unsigned long expires = peer->pmtu_expires;

H
Hiroaki SHIMODA 已提交
1536
	if (time_before(jiffies, expires)) {
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546
		u32 orig_dst_mtu = dst_mtu(dst);
		if (peer->pmtu_learned < orig_dst_mtu) {
			if (!peer->pmtu_orig)
				peer->pmtu_orig = dst_metric_raw(dst, RTAX_MTU);
			dst_metric_set(dst, RTAX_MTU, peer->pmtu_learned);
		}
	} else if (cmpxchg(&peer->pmtu_expires, expires, 0) == expires)
		dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
}

L
Linus Torvalds 已提交
1547 1548
static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
{
1549 1550 1551 1552 1553 1554
	struct rtable *rt = (struct rtable *) dst;
	struct inet_peer *peer;

	dst_confirm(dst);

	if (!rt->peer)
1555
		rt_bind_peer(rt, rt->rt_dst, 1);
1556 1557 1558
	peer = rt->peer;
	if (peer) {
		if (mtu < ip_rt_min_pmtu)
L
Linus Torvalds 已提交
1559
			mtu = ip_rt_min_pmtu;
1560
		if (!peer->pmtu_expires || mtu < peer->pmtu_learned) {
H
Hiroaki SHIMODA 已提交
1561 1562 1563 1564 1565 1566
			unsigned long pmtu_expires;

			pmtu_expires = jiffies + ip_rt_mtu_expires;
			if (!pmtu_expires)
				pmtu_expires = 1UL;

1567
			peer->pmtu_learned = mtu;
H
Hiroaki SHIMODA 已提交
1568
			peer->pmtu_expires = pmtu_expires;
1569 1570 1571

			atomic_inc(&__rt_peer_genid);
			rt->rt_peer_genid = rt_peer_genid();
L
Linus Torvalds 已提交
1572
		}
H
Hiroaki SHIMODA 已提交
1573
		check_peer_pmtu(dst, peer);
L
Linus Torvalds 已提交
1574 1575 1576
	}
}

1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601
static int check_peer_redir(struct dst_entry *dst, struct inet_peer *peer)
{
	struct rtable *rt = (struct rtable *) dst;
	__be32 orig_gw = rt->rt_gateway;

	dst_confirm(&rt->dst);

	neigh_release(rt->dst.neighbour);
	rt->dst.neighbour = NULL;

	rt->rt_gateway = peer->redirect_learned.a4;
	if (arp_bind_neighbour(&rt->dst) ||
	    !(rt->dst.neighbour->nud_state & NUD_VALID)) {
		if (rt->dst.neighbour)
			neigh_event_send(rt->dst.neighbour, NULL);
		rt->rt_gateway = orig_gw;
		return -EAGAIN;
	} else {
		rt->rt_flags |= RTCF_REDIRECTED;
		call_netevent_notifiers(NETEVENT_NEIGH_UPDATE,
					rt->dst.neighbour);
	}
	return 0;
}

L
Linus Torvalds 已提交
1602 1603
static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
{
1604 1605 1606
	struct rtable *rt = (struct rtable *) dst;

	if (rt_is_expired(rt))
T
Timo Teräs 已提交
1607
		return NULL;
1608
	if (rt->rt_peer_genid != rt_peer_genid()) {
1609 1610
		struct inet_peer *peer;

1611
		if (!rt->peer)
1612
			rt_bind_peer(rt, rt->rt_dst, 0);
1613

1614 1615 1616 1617
		peer = rt->peer;
		if (peer && peer->pmtu_expires)
			check_peer_pmtu(dst, peer);

1618 1619 1620 1621 1622 1623
		if (peer && peer->redirect_learned.a4 &&
		    peer->redirect_learned.a4 != rt->rt_gateway) {
			if (check_peer_redir(dst, peer))
				return NULL;
		}

1624 1625
		rt->rt_peer_genid = rt_peer_genid();
	}
T
Timo Teräs 已提交
1626
	return dst;
L
Linus Torvalds 已提交
1627 1628 1629 1630 1631 1632 1633
}

static void ipv4_dst_destroy(struct dst_entry *dst)
{
	struct rtable *rt = (struct rtable *) dst;
	struct inet_peer *peer = rt->peer;

1634 1635 1636 1637
	if (rt->fi) {
		fib_info_put(rt->fi);
		rt->fi = NULL;
	}
L
Linus Torvalds 已提交
1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
	if (peer) {
		rt->peer = NULL;
		inet_putpeer(peer);
	}
}


static void ipv4_link_failure(struct sk_buff *skb)
{
	struct rtable *rt;

	icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);

E
Eric Dumazet 已提交
1651
	rt = skb_rtable(skb);
1652 1653 1654 1655 1656 1657 1658 1659
	if (rt &&
	    rt->peer &&
	    rt->peer->pmtu_expires) {
		unsigned long orig = rt->peer->pmtu_expires;

		if (cmpxchg(&rt->peer->pmtu_expires, orig, 0) == orig)
			dst_metric_set(&rt->dst, RTAX_MTU, rt->peer->pmtu_orig);
	}
L
Linus Torvalds 已提交
1660 1661 1662 1663
}

static int ip_rt_bug(struct sk_buff *skb)
{
1664 1665
	printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
		&ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
L
Linus Torvalds 已提交
1666 1667
		skb->dev ? skb->dev->name : "?");
	kfree_skb(skb);
1668
	WARN_ON(1);
L
Linus Torvalds 已提交
1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
	return 0;
}

/*
   We do not cache source address of outgoing interface,
   because it is used only by IP RR, TS and SRR options,
   so that it out of fast path.

   BTW remember: "addr" is allowed to be not aligned
   in IP options!
 */

1681
void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
L
Linus Torvalds 已提交
1682
{
A
Al Viro 已提交
1683
	__be32 src;
L
Linus Torvalds 已提交
1684

1685
	if (rt_is_output_route(rt))
1686
		src = ip_hdr(skb)->saddr;
E
Eric Dumazet 已提交
1687
	else {
1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
		struct fib_result res;
		struct flowi4 fl4;
		struct iphdr *iph;

		iph = ip_hdr(skb);

		memset(&fl4, 0, sizeof(fl4));
		fl4.daddr = iph->daddr;
		fl4.saddr = iph->saddr;
		fl4.flowi4_tos = iph->tos;
		fl4.flowi4_oif = rt->dst.dev->ifindex;
		fl4.flowi4_iif = skb->dev->ifindex;
		fl4.flowi4_mark = skb->mark;
1701

E
Eric Dumazet 已提交
1702
		rcu_read_lock();
1703
		if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1704
			src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
E
Eric Dumazet 已提交
1705 1706
		else
			src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
L
Linus Torvalds 已提交
1707
					RT_SCOPE_UNIVERSE);
E
Eric Dumazet 已提交
1708 1709
		rcu_read_unlock();
	}
L
Linus Torvalds 已提交
1710 1711 1712
	memcpy(addr, &src, 4);
}

1713
#ifdef CONFIG_IP_ROUTE_CLASSID
L
Linus Torvalds 已提交
1714 1715
static void set_class_tag(struct rtable *rt, u32 tag)
{
1716 1717 1718 1719
	if (!(rt->dst.tclassid & 0xFFFF))
		rt->dst.tclassid |= tag & 0xFFFF;
	if (!(rt->dst.tclassid & 0xFFFF0000))
		rt->dst.tclassid |= tag & 0xFFFF0000;
L
Linus Torvalds 已提交
1720 1721 1722
}
#endif

1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735
static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
{
	unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);

	if (advmss == 0) {
		advmss = max_t(unsigned int, dst->dev->mtu - 40,
			       ip_rt_min_advmss);
		if (advmss > 65535 - 40)
			advmss = 65535 - 40;
	}
	return advmss;
}

1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
static unsigned int ipv4_default_mtu(const struct dst_entry *dst)
{
	unsigned int mtu = dst->dev->mtu;

	if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
		const struct rtable *rt = (const struct rtable *) dst;

		if (rt->rt_gateway != rt->rt_dst && mtu > 576)
			mtu = 576;
	}

	if (mtu > IP_MAX_MTU)
		mtu = IP_MAX_MTU;

	return mtu;
}

1753
static void rt_init_metrics(struct rtable *rt, const struct flowi4 *fl4,
1754
			    struct fib_info *fi)
D
David S. Miller 已提交
1755
{
1756 1757
	struct inet_peer *peer;
	int create = 0;
D
David S. Miller 已提交
1758

1759 1760 1761
	/* If a peer entry exists for this destination, we must hook
	 * it up in order to get at cached metrics.
	 */
1762
	if (fl4 && (fl4->flowi4_flags & FLOWI_FLAG_PRECOW_METRICS))
1763 1764
		create = 1;

1765
	rt->peer = peer = inet_getpeer_v4(rt->rt_dst, create);
1766
	if (peer) {
1767
		rt->rt_peer_genid = rt_peer_genid();
D
David S. Miller 已提交
1768 1769 1770 1771
		if (inet_metrics_new(peer))
			memcpy(peer->metrics, fi->fib_metrics,
			       sizeof(u32) * RTAX_MAX);
		dst_init_metrics(&rt->dst, peer->metrics, false);
1772 1773 1774

		if (peer->pmtu_expires)
			check_peer_pmtu(&rt->dst, peer);
1775 1776 1777 1778 1779
		if (peer->redirect_learned.a4 &&
		    peer->redirect_learned.a4 != rt->rt_gateway) {
			rt->rt_gateway = peer->redirect_learned.a4;
			rt->rt_flags |= RTCF_REDIRECTED;
		}
1780 1781 1782 1783 1784 1785
	} else {
		if (fi->fib_metrics != (u32 *) dst_default_metrics) {
			rt->fi = fi;
			atomic_inc(&fi->fib_clntref);
		}
		dst_init_metrics(&rt->dst, fi->fib_metrics, true);
D
David S. Miller 已提交
1786 1787 1788
	}
}

1789
static void rt_set_nexthop(struct rtable *rt, const struct flowi4 *fl4,
1790
			   const struct fib_result *res,
1791
			   struct fib_info *fi, u16 type, u32 itag)
L
Linus Torvalds 已提交
1792
{
1793
	struct dst_entry *dst = &rt->dst;
L
Linus Torvalds 已提交
1794 1795 1796 1797 1798

	if (fi) {
		if (FIB_RES_GW(*res) &&
		    FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
			rt->rt_gateway = FIB_RES_GW(*res);
1799
		rt_init_metrics(rt, fl4, fi);
1800
#ifdef CONFIG_IP_ROUTE_CLASSID
1801
		dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
L
Linus Torvalds 已提交
1802
#endif
1803
	}
1804 1805 1806

	if (dst_mtu(dst) > IP_MAX_MTU)
		dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
1807
	if (dst_metric_raw(dst, RTAX_ADVMSS) > 65535 - 40)
1808
		dst_metric_set(dst, RTAX_ADVMSS, 65535 - 40);
L
Linus Torvalds 已提交
1809

1810
#ifdef CONFIG_IP_ROUTE_CLASSID
L
Linus Torvalds 已提交
1811 1812 1813 1814 1815 1816 1817
#ifdef CONFIG_IP_MULTIPLE_TABLES
	set_class_tag(rt, fib_rules_tclass(res));
#endif
	set_class_tag(rt, itag);
#endif
}

1818 1819
static struct rtable *rt_dst_alloc(struct net_device *dev,
				   bool nopolicy, bool noxfrm)
1820
{
1821 1822 1823 1824
	return dst_alloc(&ipv4_dst_ops, dev, 1, -1,
			 DST_HOST |
			 (nopolicy ? DST_NOPOLICY : 0) |
			 (noxfrm ? DST_NOXFRM : 0));
1825 1826
}

1827
/* called in rcu_read_lock() section */
A
Al Viro 已提交
1828
static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
L
Linus Torvalds 已提交
1829 1830
				u8 tos, struct net_device *dev, int our)
{
1831
	unsigned int hash;
L
Linus Torvalds 已提交
1832
	struct rtable *rth;
A
Al Viro 已提交
1833
	__be32 spec_dst;
1834
	struct in_device *in_dev = __in_dev_get_rcu(dev);
L
Linus Torvalds 已提交
1835
	u32 itag = 0;
1836
	int err;
L
Linus Torvalds 已提交
1837 1838 1839 1840 1841 1842

	/* Primary sanity checks. */

	if (in_dev == NULL)
		return -EINVAL;

1843
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1844
	    ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
L
Linus Torvalds 已提交
1845 1846
		goto e_inval;

1847 1848
	if (ipv4_is_zeronet(saddr)) {
		if (!ipv4_is_local_multicast(daddr))
L
Linus Torvalds 已提交
1849 1850
			goto e_inval;
		spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
1851
	} else {
1852 1853
		err = fib_validate_source(skb, saddr, 0, tos, 0, dev, &spec_dst,
					  &itag);
1854 1855 1856
		if (err < 0)
			goto e_err;
	}
1857 1858
	rth = rt_dst_alloc(init_net.loopback_dev,
			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
L
Linus Torvalds 已提交
1859 1860 1861
	if (!rth)
		goto e_nobufs;

1862 1863 1864
#ifdef CONFIG_IP_ROUTE_CLASSID
	rth->dst.tclassid = itag;
#endif
1865
	rth->dst.output = ip_rt_bug;
L
Linus Torvalds 已提交
1866

1867 1868
	rth->rt_key_dst	= daddr;
	rth->rt_key_src	= saddr;
1869 1870 1871
	rth->rt_genid	= rt_genid(dev_net(dev));
	rth->rt_flags	= RTCF_MULTICAST;
	rth->rt_type	= RTN_MULTICAST;
1872
	rth->rt_key_tos	= tos;
1873
	rth->rt_dst	= daddr;
L
Linus Torvalds 已提交
1874
	rth->rt_src	= saddr;
1875
	rth->rt_route_iif = dev->ifindex;
1876 1877
	rth->rt_iif	= dev->ifindex;
	rth->rt_oif	= 0;
1878
	rth->rt_mark    = skb->mark;
L
Linus Torvalds 已提交
1879 1880
	rth->rt_gateway	= daddr;
	rth->rt_spec_dst= spec_dst;
1881 1882 1883
	rth->rt_peer_genid = 0;
	rth->peer = NULL;
	rth->fi = NULL;
L
Linus Torvalds 已提交
1884
	if (our) {
1885
		rth->dst.input= ip_local_deliver;
L
Linus Torvalds 已提交
1886 1887 1888 1889
		rth->rt_flags |= RTCF_LOCAL;
	}

#ifdef CONFIG_IP_MROUTE
1890
	if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1891
		rth->dst.input = ip_mr_input;
L
Linus Torvalds 已提交
1892 1893 1894
#endif
	RT_CACHE_STAT_INC(in_slow_mc);

1895
	hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1896 1897 1898 1899
	rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
	err = 0;
	if (IS_ERR(rth))
		err = PTR_ERR(rth);
L
Linus Torvalds 已提交
1900 1901 1902 1903

e_nobufs:
	return -ENOBUFS;
e_inval:
1904
	return -EINVAL;
1905 1906
e_err:
	return err;
L
Linus Torvalds 已提交
1907 1908 1909 1910 1911 1912
}


static void ip_handle_martian_source(struct net_device *dev,
				     struct in_device *in_dev,
				     struct sk_buff *skb,
A
Al Viro 已提交
1913 1914
				     __be32 daddr,
				     __be32 saddr)
L
Linus Torvalds 已提交
1915 1916 1917 1918 1919 1920 1921 1922
{
	RT_CACHE_STAT_INC(in_martian_src);
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
		/*
		 *	RFC1812 recommendation, if source is martian,
		 *	the only hint is MAC header.
		 */
1923 1924
		printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
			&daddr, &saddr, dev->name);
1925
		if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
L
Linus Torvalds 已提交
1926
			int i;
1927
			const unsigned char *p = skb_mac_header(skb);
L
Linus Torvalds 已提交
1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
			printk(KERN_WARNING "ll header: ");
			for (i = 0; i < dev->hard_header_len; i++, p++) {
				printk("%02x", *p);
				if (i < (dev->hard_header_len - 1))
					printk(":");
			}
			printk("\n");
		}
	}
#endif
}

1940
/* called in rcu_read_lock() section */
S
Stephen Hemminger 已提交
1941
static int __mkroute_input(struct sk_buff *skb,
1942
			   const struct fib_result *res,
S
Stephen Hemminger 已提交
1943 1944 1945
			   struct in_device *in_dev,
			   __be32 daddr, __be32 saddr, u32 tos,
			   struct rtable **result)
L
Linus Torvalds 已提交
1946 1947 1948 1949
{
	struct rtable *rth;
	int err;
	struct in_device *out_dev;
1950
	unsigned int flags = 0;
1951 1952
	__be32 spec_dst;
	u32 itag;
L
Linus Torvalds 已提交
1953 1954

	/* get a working reference to the output device */
1955
	out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
L
Linus Torvalds 已提交
1956 1957 1958 1959 1960 1961 1962 1963
	if (out_dev == NULL) {
		if (net_ratelimit())
			printk(KERN_CRIT "Bug in ip_route_input" \
			       "_slow(). Please, report\n");
		return -EINVAL;
	}


1964 1965
	err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
				  in_dev->dev, &spec_dst, &itag);
L
Linus Torvalds 已提交
1966
	if (err < 0) {
1967
		ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
L
Linus Torvalds 已提交
1968
					 saddr);
1969

L
Linus Torvalds 已提交
1970 1971 1972 1973 1974 1975
		goto cleanup;
	}

	if (err)
		flags |= RTCF_DIRECTSRC;

1976
	if (out_dev == in_dev && err &&
L
Linus Torvalds 已提交
1977 1978 1979 1980 1981 1982 1983
	    (IN_DEV_SHARED_MEDIA(out_dev) ||
	     inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
		flags |= RTCF_DOREDIRECT;

	if (skb->protocol != htons(ETH_P_IP)) {
		/* Not IP (i.e. ARP). Do not create route, if it is
		 * invalid for proxy arp. DNAT routes are always valid.
1984 1985 1986 1987
		 *
		 * Proxy arp feature have been extended to allow, ARP
		 * replies back to the same interface, to support
		 * Private VLAN switch technologies. See arp.c.
L
Linus Torvalds 已提交
1988
		 */
1989 1990
		if (out_dev == in_dev &&
		    IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
L
Linus Torvalds 已提交
1991 1992 1993 1994 1995
			err = -EINVAL;
			goto cleanup;
		}
	}

1996 1997
	rth = rt_dst_alloc(out_dev->dev,
			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
1998
			   IN_DEV_CONF_GET(out_dev, NOXFRM));
L
Linus Torvalds 已提交
1999 2000 2001 2002 2003
	if (!rth) {
		err = -ENOBUFS;
		goto cleanup;
	}

2004 2005
	rth->rt_key_dst	= daddr;
	rth->rt_key_src	= saddr;
2006 2007 2008
	rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
	rth->rt_flags = flags;
	rth->rt_type = res->type;
2009
	rth->rt_key_tos	= tos;
2010
	rth->rt_dst	= daddr;
L
Linus Torvalds 已提交
2011
	rth->rt_src	= saddr;
2012
	rth->rt_route_iif = in_dev->dev->ifindex;
2013 2014
	rth->rt_iif 	= in_dev->dev->ifindex;
	rth->rt_oif 	= 0;
2015 2016
	rth->rt_mark    = skb->mark;
	rth->rt_gateway	= daddr;
L
Linus Torvalds 已提交
2017
	rth->rt_spec_dst= spec_dst;
2018 2019 2020
	rth->rt_peer_genid = 0;
	rth->peer = NULL;
	rth->fi = NULL;
L
Linus Torvalds 已提交
2021

2022 2023
	rth->dst.input = ip_forward;
	rth->dst.output = ip_output;
L
Linus Torvalds 已提交
2024

2025
	rt_set_nexthop(rth, NULL, res, res->fi, res->type, itag);
L
Linus Torvalds 已提交
2026 2027 2028 2029 2030

	*result = rth;
	err = 0;
 cleanup:
	return err;
2031
}
L
Linus Torvalds 已提交
2032

S
Stephen Hemminger 已提交
2033 2034
static int ip_mkroute_input(struct sk_buff *skb,
			    struct fib_result *res,
2035
			    const struct flowi4 *fl4,
S
Stephen Hemminger 已提交
2036 2037
			    struct in_device *in_dev,
			    __be32 daddr, __be32 saddr, u32 tos)
L
Linus Torvalds 已提交
2038
{
C
Chuck Short 已提交
2039
	struct rtable* rth = NULL;
L
Linus Torvalds 已提交
2040 2041 2042 2043
	int err;
	unsigned hash;

#ifdef CONFIG_IP_ROUTE_MULTIPATH
2044
	if (res->fi && res->fi->fib_nhs > 1)
2045
		fib_select_multipath(res);
L
Linus Torvalds 已提交
2046 2047 2048 2049 2050 2051 2052 2053
#endif

	/* create a routing cache entry */
	err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
	if (err)
		return err;

	/* put it into the cache */
2054
	hash = rt_hash(daddr, saddr, fl4->flowi4_iif,
2055
		       rt_genid(dev_net(rth->dst.dev)));
2056
	rth = rt_intern_hash(hash, rth, skb, fl4->flowi4_iif);
2057 2058 2059
	if (IS_ERR(rth))
		return PTR_ERR(rth);
	return 0;
L
Linus Torvalds 已提交
2060 2061 2062 2063 2064 2065 2066 2067 2068 2069
}

/*
 *	NOTE. We drop all the packets that has local source
 *	addresses, because every properly looped back packet
 *	must have correct destination already attached by output routine.
 *
 *	Such approach solves two big problems:
 *	1. Not simplex devices are handled properly.
 *	2. IP spoofing attempts are filtered with 100% of guarantee.
E
Eric Dumazet 已提交
2070
 *	called with rcu_read_lock()
L
Linus Torvalds 已提交
2071 2072
 */

A
Al Viro 已提交
2073
static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
L
Linus Torvalds 已提交
2074 2075 2076
			       u8 tos, struct net_device *dev)
{
	struct fib_result res;
2077
	struct in_device *in_dev = __in_dev_get_rcu(dev);
2078
	struct flowi4	fl4;
L
Linus Torvalds 已提交
2079 2080 2081 2082
	unsigned	flags = 0;
	u32		itag = 0;
	struct rtable * rth;
	unsigned	hash;
A
Al Viro 已提交
2083
	__be32		spec_dst;
L
Linus Torvalds 已提交
2084
	int		err = -EINVAL;
2085
	struct net    * net = dev_net(dev);
L
Linus Torvalds 已提交
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095

	/* IP on this device is disabled. */

	if (!in_dev)
		goto out;

	/* Check for the most weird martians, which can be not detected
	   by fib_lookup.
	 */

2096
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
2097
	    ipv4_is_loopback(saddr))
L
Linus Torvalds 已提交
2098 2099
		goto martian_source;

2100
	if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
L
Linus Torvalds 已提交
2101 2102 2103 2104 2105
		goto brd_input;

	/* Accept zero addresses only to limited broadcast;
	 * I even do not know to fix it or not. Waiting for complains :-)
	 */
2106
	if (ipv4_is_zeronet(saddr))
L
Linus Torvalds 已提交
2107 2108
		goto martian_source;

2109
	if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr))
L
Linus Torvalds 已提交
2110 2111 2112 2113 2114
		goto martian_destination;

	/*
	 *	Now we are ready to route packet.
	 */
2115 2116 2117 2118 2119 2120 2121 2122
	fl4.flowi4_oif = 0;
	fl4.flowi4_iif = dev->ifindex;
	fl4.flowi4_mark = skb->mark;
	fl4.flowi4_tos = tos;
	fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
	fl4.daddr = daddr;
	fl4.saddr = saddr;
	err = fib_lookup(net, &fl4, &res);
E
Eric Dumazet 已提交
2123
	if (err != 0) {
L
Linus Torvalds 已提交
2124
		if (!IN_DEV_FORWARD(in_dev))
2125
			goto e_hostunreach;
L
Linus Torvalds 已提交
2126 2127 2128 2129 2130 2131 2132 2133 2134
		goto no_route;
	}

	RT_CACHE_STAT_INC(in_slow_tot);

	if (res.type == RTN_BROADCAST)
		goto brd_input;

	if (res.type == RTN_LOCAL) {
2135
		err = fib_validate_source(skb, saddr, daddr, tos,
E
Eric Dumazet 已提交
2136
					  net->loopback_dev->ifindex,
2137
					  dev, &spec_dst, &itag);
2138 2139 2140
		if (err < 0)
			goto martian_source_keep_err;
		if (err)
L
Linus Torvalds 已提交
2141 2142 2143 2144 2145 2146
			flags |= RTCF_DIRECTSRC;
		spec_dst = daddr;
		goto local_input;
	}

	if (!IN_DEV_FORWARD(in_dev))
2147
		goto e_hostunreach;
L
Linus Torvalds 已提交
2148 2149 2150
	if (res.type != RTN_UNICAST)
		goto martian_destination;

2151
	err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
L
Linus Torvalds 已提交
2152 2153 2154 2155 2156 2157
out:	return err;

brd_input:
	if (skb->protocol != htons(ETH_P_IP))
		goto e_inval;

2158
	if (ipv4_is_zeronet(saddr))
L
Linus Torvalds 已提交
2159 2160
		spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
	else {
2161 2162
		err = fib_validate_source(skb, saddr, 0, tos, 0, dev, &spec_dst,
					  &itag);
L
Linus Torvalds 已提交
2163
		if (err < 0)
2164
			goto martian_source_keep_err;
L
Linus Torvalds 已提交
2165 2166 2167 2168 2169 2170 2171 2172
		if (err)
			flags |= RTCF_DIRECTSRC;
	}
	flags |= RTCF_BROADCAST;
	res.type = RTN_BROADCAST;
	RT_CACHE_STAT_INC(in_brd);

local_input:
2173 2174
	rth = rt_dst_alloc(net->loopback_dev,
			   IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
L
Linus Torvalds 已提交
2175 2176 2177
	if (!rth)
		goto e_nobufs;

2178
	rth->dst.input= ip_local_deliver;
2179
	rth->dst.output= ip_rt_bug;
2180 2181 2182
#ifdef CONFIG_IP_ROUTE_CLASSID
	rth->dst.tclassid = itag;
#endif
L
Linus Torvalds 已提交
2183

2184 2185
	rth->rt_key_dst	= daddr;
	rth->rt_key_src	= saddr;
2186 2187 2188
	rth->rt_genid = rt_genid(net);
	rth->rt_flags 	= flags|RTCF_LOCAL;
	rth->rt_type	= res.type;
2189
	rth->rt_key_tos	= tos;
2190
	rth->rt_dst	= daddr;
L
Linus Torvalds 已提交
2191
	rth->rt_src	= saddr;
2192
#ifdef CONFIG_IP_ROUTE_CLASSID
2193
	rth->dst.tclassid = itag;
L
Linus Torvalds 已提交
2194
#endif
2195
	rth->rt_route_iif = dev->ifindex;
2196
	rth->rt_iif	= dev->ifindex;
2197 2198
	rth->rt_oif	= 0;
	rth->rt_mark    = skb->mark;
L
Linus Torvalds 已提交
2199 2200
	rth->rt_gateway	= daddr;
	rth->rt_spec_dst= spec_dst;
2201 2202 2203
	rth->rt_peer_genid = 0;
	rth->peer = NULL;
	rth->fi = NULL;
L
Linus Torvalds 已提交
2204
	if (res.type == RTN_UNREACHABLE) {
2205 2206
		rth->dst.input= ip_error;
		rth->dst.error= -err;
L
Linus Torvalds 已提交
2207 2208
		rth->rt_flags 	&= ~RTCF_LOCAL;
	}
2209 2210
	hash = rt_hash(daddr, saddr, fl4.flowi4_iif, rt_genid(net));
	rth = rt_intern_hash(hash, rth, skb, fl4.flowi4_iif);
2211 2212 2213
	err = 0;
	if (IS_ERR(rth))
		err = PTR_ERR(rth);
E
Eric Dumazet 已提交
2214
	goto out;
L
Linus Torvalds 已提交
2215 2216 2217 2218 2219

no_route:
	RT_CACHE_STAT_INC(in_no_route);
	spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
	res.type = RTN_UNREACHABLE;
2220 2221
	if (err == -ESRCH)
		err = -ENETUNREACH;
L
Linus Torvalds 已提交
2222 2223 2224 2225 2226 2227 2228 2229 2230
	goto local_input;

	/*
	 *	Do not cache martian addresses: they should be logged (RFC1812)
	 */
martian_destination:
	RT_CACHE_STAT_INC(in_martian_dst);
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
2231 2232
		printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
			&daddr, &saddr, dev->name);
L
Linus Torvalds 已提交
2233
#endif
2234 2235

e_hostunreach:
2236
	err = -EHOSTUNREACH;
E
Eric Dumazet 已提交
2237
	goto out;
2238

L
Linus Torvalds 已提交
2239 2240
e_inval:
	err = -EINVAL;
E
Eric Dumazet 已提交
2241
	goto out;
L
Linus Torvalds 已提交
2242 2243 2244

e_nobufs:
	err = -ENOBUFS;
E
Eric Dumazet 已提交
2245
	goto out;
L
Linus Torvalds 已提交
2246 2247

martian_source:
2248 2249
	err = -EINVAL;
martian_source_keep_err:
L
Linus Torvalds 已提交
2250
	ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
E
Eric Dumazet 已提交
2251
	goto out;
L
Linus Torvalds 已提交
2252 2253
}

2254 2255
int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
			   u8 tos, struct net_device *dev, bool noref)
L
Linus Torvalds 已提交
2256 2257 2258 2259
{
	struct rtable * rth;
	unsigned	hash;
	int iif = dev->ifindex;
2260
	struct net *net;
2261
	int res;
L
Linus Torvalds 已提交
2262

2263
	net = dev_net(dev);
2264

2265 2266
	rcu_read_lock();

2267 2268 2269
	if (!rt_caching(net))
		goto skip_cache;

L
Linus Torvalds 已提交
2270
	tos &= IPTOS_RT_MASK;
2271
	hash = rt_hash(daddr, saddr, iif, rt_genid(net));
L
Linus Torvalds 已提交
2272 2273

	for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2274
	     rth = rcu_dereference(rth->dst.rt_next)) {
2275 2276 2277 2278
		if ((((__force u32)rth->rt_key_dst ^ (__force u32)daddr) |
		     ((__force u32)rth->rt_key_src ^ (__force u32)saddr) |
		     (rth->rt_iif ^ iif) |
		     rth->rt_oif |
2279
		     (rth->rt_key_tos ^ tos)) == 0 &&
2280
		    rth->rt_mark == skb->mark &&
2281
		    net_eq(dev_net(rth->dst.dev), net) &&
2282
		    !rt_is_expired(rth)) {
2283
			if (noref) {
2284 2285
				dst_use_noref(&rth->dst, jiffies);
				skb_dst_set_noref(skb, &rth->dst);
2286
			} else {
2287 2288
				dst_use(&rth->dst, jiffies);
				skb_dst_set(skb, &rth->dst);
2289
			}
L
Linus Torvalds 已提交
2290 2291 2292 2293 2294 2295 2296
			RT_CACHE_STAT_INC(in_hit);
			rcu_read_unlock();
			return 0;
		}
		RT_CACHE_STAT_INC(in_hlist_search);
	}

2297
skip_cache:
L
Linus Torvalds 已提交
2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308
	/* Multicast recognition logic is moved from route cache to here.
	   The problem was that too many Ethernet cards have broken/missing
	   hardware multicast filters :-( As result the host on multicasting
	   network acquires a lot of useless route cache entries, sort of
	   SDR messages from all the world. Now we try to get rid of them.
	   Really, provided software IP multicast filter is organized
	   reasonably (at least, hashed), it does not result in a slowdown
	   comparing with route cache reject entries.
	   Note, that multicast routers are not affected, because
	   route cache entry is created eventually.
	 */
2309
	if (ipv4_is_multicast(daddr)) {
2310
		struct in_device *in_dev = __in_dev_get_rcu(dev);
L
Linus Torvalds 已提交
2311

2312
		if (in_dev) {
2313 2314
			int our = ip_check_mc_rcu(in_dev, daddr, saddr,
						  ip_hdr(skb)->protocol);
L
Linus Torvalds 已提交
2315 2316
			if (our
#ifdef CONFIG_IP_MROUTE
2317 2318 2319
				||
			    (!ipv4_is_local_multicast(daddr) &&
			     IN_DEV_MFORWARD(in_dev))
L
Linus Torvalds 已提交
2320
#endif
2321
			   ) {
2322 2323
				int res = ip_route_input_mc(skb, daddr, saddr,
							    tos, dev, our);
L
Linus Torvalds 已提交
2324
				rcu_read_unlock();
2325
				return res;
L
Linus Torvalds 已提交
2326 2327 2328 2329 2330
			}
		}
		rcu_read_unlock();
		return -EINVAL;
	}
2331 2332 2333
	res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
	rcu_read_unlock();
	return res;
L
Linus Torvalds 已提交
2334
}
2335
EXPORT_SYMBOL(ip_route_input_common);
L
Linus Torvalds 已提交
2336

E
Eric Dumazet 已提交
2337
/* called with rcu_read_lock() */
2338
static struct rtable *__mkroute_output(const struct fib_result *res,
2339
				       const struct flowi4 *fl4,
2340 2341
				       __be32 orig_daddr, __be32 orig_saddr,
				       int orig_oif, struct net_device *dev_out,
2342
				       unsigned int flags)
L
Linus Torvalds 已提交
2343
{
2344
	struct fib_info *fi = res->fi;
2345
	u32 tos = RT_FL_TOS(fl4);
2346
	struct in_device *in_dev;
2347
	u16 type = res->type;
2348
	struct rtable *rth;
L
Linus Torvalds 已提交
2349

2350
	if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2351
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
2352

2353
	if (ipv4_is_lbcast(fl4->daddr))
2354
		type = RTN_BROADCAST;
2355
	else if (ipv4_is_multicast(fl4->daddr))
2356
		type = RTN_MULTICAST;
2357
	else if (ipv4_is_zeronet(fl4->daddr))
2358
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
2359 2360 2361 2362

	if (dev_out->flags & IFF_LOOPBACK)
		flags |= RTCF_LOCAL;

E
Eric Dumazet 已提交
2363
	in_dev = __in_dev_get_rcu(dev_out);
E
Eric Dumazet 已提交
2364
	if (!in_dev)
2365
		return ERR_PTR(-EINVAL);
E
Eric Dumazet 已提交
2366

2367
	if (type == RTN_BROADCAST) {
L
Linus Torvalds 已提交
2368
		flags |= RTCF_BROADCAST | RTCF_LOCAL;
2369 2370
		fi = NULL;
	} else if (type == RTN_MULTICAST) {
E
Eric Dumazet 已提交
2371
		flags |= RTCF_MULTICAST | RTCF_LOCAL;
2372 2373
		if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
				     fl4->flowi4_proto))
L
Linus Torvalds 已提交
2374 2375
			flags &= ~RTCF_LOCAL;
		/* If multicast route do not exist use
E
Eric Dumazet 已提交
2376 2377
		 * default one, but do not gateway in this case.
		 * Yes, it is hack.
L
Linus Torvalds 已提交
2378
		 */
2379 2380
		if (fi && res->prefixlen < 4)
			fi = NULL;
L
Linus Torvalds 已提交
2381 2382
	}

2383 2384
	rth = rt_dst_alloc(dev_out,
			   IN_DEV_CONF_GET(in_dev, NOPOLICY),
2385
			   IN_DEV_CONF_GET(in_dev, NOXFRM));
2386
	if (!rth)
2387
		return ERR_PTR(-ENOBUFS);
2388

2389 2390
	rth->dst.output = ip_output;

2391 2392
	rth->rt_key_dst	= orig_daddr;
	rth->rt_key_src	= orig_saddr;
2393 2394 2395
	rth->rt_genid = rt_genid(dev_net(dev_out));
	rth->rt_flags	= flags;
	rth->rt_type	= type;
2396
	rth->rt_key_tos	= tos;
2397 2398
	rth->rt_dst	= fl4->daddr;
	rth->rt_src	= fl4->saddr;
2399
	rth->rt_route_iif = 0;
2400 2401 2402
	rth->rt_iif	= orig_oif ? : dev_out->ifindex;
	rth->rt_oif	= orig_oif;
	rth->rt_mark    = fl4->flowi4_mark;
2403 2404
	rth->rt_gateway = fl4->daddr;
	rth->rt_spec_dst= fl4->saddr;
2405 2406 2407
	rth->rt_peer_genid = 0;
	rth->peer = NULL;
	rth->fi = NULL;
L
Linus Torvalds 已提交
2408 2409 2410 2411

	RT_CACHE_STAT_INC(out_slow_tot);

	if (flags & RTCF_LOCAL) {
2412
		rth->dst.input = ip_local_deliver;
2413
		rth->rt_spec_dst = fl4->daddr;
L
Linus Torvalds 已提交
2414 2415
	}
	if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2416
		rth->rt_spec_dst = fl4->saddr;
2417
		if (flags & RTCF_LOCAL &&
L
Linus Torvalds 已提交
2418
		    !(dev_out->flags & IFF_LOOPBACK)) {
2419
			rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2420 2421 2422
			RT_CACHE_STAT_INC(out_slow_mc);
		}
#ifdef CONFIG_IP_MROUTE
2423
		if (type == RTN_MULTICAST) {
L
Linus Torvalds 已提交
2424
			if (IN_DEV_MFORWARD(in_dev) &&
2425
			    !ipv4_is_local_multicast(fl4->daddr)) {
2426 2427
				rth->dst.input = ip_mr_input;
				rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2428 2429 2430 2431 2432
			}
		}
#endif
	}

2433
	rt_set_nexthop(rth, fl4, res, fi, type, 0);
L
Linus Torvalds 已提交
2434

2435
	return rth;
L
Linus Torvalds 已提交
2436 2437 2438 2439
}

/*
 * Major route resolver routine.
2440
 * called with rcu_read_lock();
L
Linus Torvalds 已提交
2441 2442
 */

2443
static struct rtable *ip_route_output_slow(struct net *net, struct flowi4 *fl4)
L
Linus Torvalds 已提交
2444 2445
{
	struct net_device *dev_out = NULL;
2446 2447 2448
	u32 tos	= RT_FL_TOS(fl4);
	unsigned int flags = 0;
	struct fib_result res;
2449
	struct rtable *rth;
2450 2451 2452
	__be32 orig_daddr;
	__be32 orig_saddr;
	int orig_oif;
L
Linus Torvalds 已提交
2453 2454 2455 2456 2457 2458

	res.fi		= NULL;
#ifdef CONFIG_IP_MULTIPLE_TABLES
	res.r		= NULL;
#endif

2459 2460 2461 2462 2463 2464 2465 2466
	orig_daddr = fl4->daddr;
	orig_saddr = fl4->saddr;
	orig_oif = fl4->flowi4_oif;

	fl4->flowi4_iif = net->loopback_dev->ifindex;
	fl4->flowi4_tos = tos & IPTOS_RT_MASK;
	fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
			 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2467

2468
	rcu_read_lock();
2469
	if (fl4->saddr) {
2470
		rth = ERR_PTR(-EINVAL);
2471 2472 2473
		if (ipv4_is_multicast(fl4->saddr) ||
		    ipv4_is_lbcast(fl4->saddr) ||
		    ipv4_is_zeronet(fl4->saddr))
L
Linus Torvalds 已提交
2474 2475 2476 2477
			goto out;

		/* I removed check for oif == dev_out->oif here.
		   It was wrong for two reasons:
2478 2479
		   1. ip_dev_find(net, saddr) can return wrong iface, if saddr
		      is assigned to multiple interfaces.
L
Linus Torvalds 已提交
2480 2481 2482 2483
		   2. Moreover, we are allowed to send packets with saddr
		      of another iface. --ANK
		 */

2484 2485 2486
		if (fl4->flowi4_oif == 0 &&
		    (ipv4_is_multicast(fl4->daddr) ||
		     ipv4_is_lbcast(fl4->daddr))) {
2487
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2488
			dev_out = __ip_dev_find(net, fl4->saddr, false);
2489 2490 2491
			if (dev_out == NULL)
				goto out;

L
Linus Torvalds 已提交
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506
			/* Special hack: user can direct multicasts
			   and limited broadcast via necessary interface
			   without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
			   This hack is not just for fun, it allows
			   vic,vat and friends to work.
			   They bind socket to loopback, set ttl to zero
			   and expect that it will work.
			   From the viewpoint of routing cache they are broken,
			   because we are not allowed to build multicast path
			   with loopback source addr (look, routing cache
			   cannot know, that ttl is zero, so that packet
			   will not leave this host and route is valid).
			   Luckily, this hack is good workaround.
			 */

2507
			fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2508 2509
			goto make_route;
		}
2510

2511
		if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2512
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2513
			if (!__ip_dev_find(net, fl4->saddr, false))
2514 2515
				goto out;
		}
L
Linus Torvalds 已提交
2516 2517 2518
	}


2519 2520
	if (fl4->flowi4_oif) {
		dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2521
		rth = ERR_PTR(-ENODEV);
L
Linus Torvalds 已提交
2522 2523
		if (dev_out == NULL)
			goto out;
2524 2525

		/* RACE: Check return value of inet_select_addr instead. */
2526
		if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2527
			rth = ERR_PTR(-ENETUNREACH);
2528 2529
			goto out;
		}
2530 2531 2532 2533 2534
		if (ipv4_is_local_multicast(fl4->daddr) ||
		    ipv4_is_lbcast(fl4->daddr)) {
			if (!fl4->saddr)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
L
Linus Torvalds 已提交
2535 2536
			goto make_route;
		}
2537 2538 2539 2540 2541 2542 2543
		if (fl4->saddr) {
			if (ipv4_is_multicast(fl4->daddr))
				fl4->saddr = inet_select_addr(dev_out, 0,
							      fl4->flowi4_scope);
			else if (!fl4->daddr)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_HOST);
L
Linus Torvalds 已提交
2544 2545 2546
		}
	}

2547 2548 2549 2550
	if (!fl4->daddr) {
		fl4->daddr = fl4->saddr;
		if (!fl4->daddr)
			fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2551
		dev_out = net->loopback_dev;
2552
		fl4->flowi4_oif = net->loopback_dev->ifindex;
L
Linus Torvalds 已提交
2553 2554 2555 2556 2557
		res.type = RTN_LOCAL;
		flags |= RTCF_LOCAL;
		goto make_route;
	}

2558
	if (fib_lookup(net, fl4, &res)) {
L
Linus Torvalds 已提交
2559
		res.fi = NULL;
2560
		if (fl4->flowi4_oif) {
L
Linus Torvalds 已提交
2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578
			/* Apparently, routing tables are wrong. Assume,
			   that the destination is on link.

			   WHY? DW.
			   Because we are allowed to send to iface
			   even if it has NO routes and NO assigned
			   addresses. When oif is specified, routing
			   tables are looked up with only one purpose:
			   to catch if destination is gatewayed, rather than
			   direct. Moreover, if MSG_DONTROUTE is set,
			   we send packet, ignoring both routing tables
			   and ifaddr state. --ANK


			   We could make it even if oif is unknown,
			   likely IPv6, but we do not.
			 */

2579 2580 2581
			if (fl4->saddr == 0)
				fl4->saddr = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
L
Linus Torvalds 已提交
2582 2583 2584
			res.type = RTN_UNICAST;
			goto make_route;
		}
2585
		rth = ERR_PTR(-ENETUNREACH);
L
Linus Torvalds 已提交
2586 2587 2588 2589
		goto out;
	}

	if (res.type == RTN_LOCAL) {
2590
		if (!fl4->saddr) {
2591
			if (res.fi->fib_prefsrc)
2592
				fl4->saddr = res.fi->fib_prefsrc;
2593
			else
2594
				fl4->saddr = fl4->daddr;
2595
		}
2596
		dev_out = net->loopback_dev;
2597
		fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2598 2599 2600 2601 2602 2603
		res.fi = NULL;
		flags |= RTCF_LOCAL;
		goto make_route;
	}

#ifdef CONFIG_IP_ROUTE_MULTIPATH
2604
	if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2605
		fib_select_multipath(&res);
L
Linus Torvalds 已提交
2606 2607
	else
#endif
2608 2609
	if (!res.prefixlen &&
	    res.table->tb_num_default > 1 &&
2610
	    res.type == RTN_UNICAST && !fl4->flowi4_oif)
2611
		fib_select_default(&res);
L
Linus Torvalds 已提交
2612

2613 2614
	if (!fl4->saddr)
		fl4->saddr = FIB_RES_PREFSRC(net, res);
L
Linus Torvalds 已提交
2615 2616

	dev_out = FIB_RES_DEV(res);
2617
	fl4->flowi4_oif = dev_out->ifindex;
L
Linus Torvalds 已提交
2618 2619 2620


make_route:
2621 2622
	rth = __mkroute_output(&res, fl4, orig_daddr, orig_saddr, orig_oif,
			       dev_out, flags);
2623
	if (!IS_ERR(rth)) {
2624 2625
		unsigned int hash;

2626
		hash = rt_hash(orig_daddr, orig_saddr, orig_oif,
2627
			       rt_genid(dev_net(dev_out)));
2628
		rth = rt_intern_hash(hash, rth, NULL, orig_oif);
2629
	}
L
Linus Torvalds 已提交
2630

2631 2632
out:
	rcu_read_unlock();
2633
	return rth;
L
Linus Torvalds 已提交
2634 2635
}

2636
struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *flp4)
L
Linus Torvalds 已提交
2637 2638
{
	struct rtable *rth;
2639
	unsigned int hash;
L
Linus Torvalds 已提交
2640

2641 2642 2643
	if (!rt_caching(net))
		goto slow_output;

2644
	hash = rt_hash(flp4->daddr, flp4->saddr, flp4->flowi4_oif, rt_genid(net));
L
Linus Torvalds 已提交
2645 2646

	rcu_read_lock_bh();
2647
	for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2648
		rth = rcu_dereference_bh(rth->dst.rt_next)) {
2649 2650
		if (rth->rt_key_dst == flp4->daddr &&
		    rth->rt_key_src == flp4->saddr &&
2651
		    rt_is_output_route(rth) &&
2652 2653
		    rth->rt_oif == flp4->flowi4_oif &&
		    rth->rt_mark == flp4->flowi4_mark &&
2654
		    !((rth->rt_key_tos ^ flp4->flowi4_tos) &
2655
			    (IPTOS_RT_MASK | RTO_ONLINK)) &&
2656
		    net_eq(dev_net(rth->dst.dev), net) &&
2657
		    !rt_is_expired(rth)) {
2658
			dst_use(&rth->dst, jiffies);
L
Linus Torvalds 已提交
2659 2660
			RT_CACHE_STAT_INC(out_hit);
			rcu_read_unlock_bh();
2661 2662 2663 2664
			if (!flp4->saddr)
				flp4->saddr = rth->rt_src;
			if (!flp4->daddr)
				flp4->daddr = rth->rt_dst;
2665
			return rth;
L
Linus Torvalds 已提交
2666 2667 2668 2669 2670
		}
		RT_CACHE_STAT_INC(out_hlist_search);
	}
	rcu_read_unlock_bh();

2671
slow_output:
2672
	return ip_route_output_slow(net, flp4);
L
Linus Torvalds 已提交
2673
}
2674 2675
EXPORT_SYMBOL_GPL(__ip_route_output_key);

2676 2677 2678 2679 2680
static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
{
	return NULL;
}

2681 2682 2683 2684 2685
static unsigned int ipv4_blackhole_default_mtu(const struct dst_entry *dst)
{
	return 0;
}

2686 2687 2688 2689
static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
{
}

2690 2691 2692 2693 2694 2695
static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
					  unsigned long old)
{
	return NULL;
}

2696 2697
static struct dst_ops ipv4_dst_blackhole_ops = {
	.family			=	AF_INET,
2698
	.protocol		=	cpu_to_be16(ETH_P_IP),
2699
	.destroy		=	ipv4_dst_destroy,
2700
	.check			=	ipv4_blackhole_dst_check,
2701
	.default_mtu		=	ipv4_blackhole_default_mtu,
2702
	.default_advmss		=	ipv4_default_advmss,
2703
	.update_pmtu		=	ipv4_rt_blackhole_update_pmtu,
2704
	.cow_metrics		=	ipv4_rt_blackhole_cow_metrics,
2705 2706
};

2707
struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2708
{
2709
	struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, 0, 0);
2710
	struct rtable *ort = (struct rtable *) dst_orig;
2711 2712

	if (rt) {
2713
		struct dst_entry *new = &rt->dst;
2714 2715

		new->__use = 1;
2716 2717
		new->input = dst_discard;
		new->output = dst_discard;
2718
		dst_copy_metrics(new, &ort->dst);
2719

2720
		new->dev = ort->dst.dev;
2721 2722 2723
		if (new->dev)
			dev_hold(new->dev);

2724 2725
		rt->rt_key_dst = ort->rt_key_dst;
		rt->rt_key_src = ort->rt_key_src;
2726
		rt->rt_key_tos = ort->rt_key_tos;
2727
		rt->rt_route_iif = ort->rt_route_iif;
2728 2729 2730
		rt->rt_iif = ort->rt_iif;
		rt->rt_oif = ort->rt_oif;
		rt->rt_mark = ort->rt_mark;
2731

2732
		rt->rt_genid = rt_genid(net);
2733 2734 2735 2736 2737 2738 2739 2740 2741
		rt->rt_flags = ort->rt_flags;
		rt->rt_type = ort->rt_type;
		rt->rt_dst = ort->rt_dst;
		rt->rt_src = ort->rt_src;
		rt->rt_gateway = ort->rt_gateway;
		rt->rt_spec_dst = ort->rt_spec_dst;
		rt->peer = ort->peer;
		if (rt->peer)
			atomic_inc(&rt->peer->refcnt);
2742 2743 2744
		rt->fi = ort->fi;
		if (rt->fi)
			atomic_inc(&rt->fi->fib_clntref);
2745 2746 2747 2748

		dst_free(new);
	}

2749 2750 2751
	dst_release(dst_orig);

	return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2752 2753
}

2754
struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2755
				    struct sock *sk)
L
Linus Torvalds 已提交
2756
{
2757
	struct rtable *rt = __ip_route_output_key(net, flp4);
L
Linus Torvalds 已提交
2758

2759 2760
	if (IS_ERR(rt))
		return rt;
L
Linus Torvalds 已提交
2761

2762
	if (flp4->flowi4_proto)
2763 2764 2765
		rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
						   flowi4_to_flowi(flp4),
						   sk, 0);
L
Linus Torvalds 已提交
2766

2767
	return rt;
L
Linus Torvalds 已提交
2768
}
2769 2770
EXPORT_SYMBOL_GPL(ip_route_output_flow);

2771 2772
static int rt_fill_info(struct net *net,
			struct sk_buff *skb, u32 pid, u32 seq, int event,
2773
			int nowait, unsigned int flags)
L
Linus Torvalds 已提交
2774
{
E
Eric Dumazet 已提交
2775
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
2776
	struct rtmsg *r;
2777
	struct nlmsghdr *nlh;
2778 2779
	long expires;
	u32 id = 0, ts = 0, tsage = 0, error;
2780 2781 2782

	nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
	if (nlh == NULL)
2783
		return -EMSGSIZE;
2784 2785

	r = nlmsg_data(nlh);
L
Linus Torvalds 已提交
2786 2787 2788
	r->rtm_family	 = AF_INET;
	r->rtm_dst_len	= 32;
	r->rtm_src_len	= 0;
2789
	r->rtm_tos	= rt->rt_key_tos;
L
Linus Torvalds 已提交
2790
	r->rtm_table	= RT_TABLE_MAIN;
2791
	NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
L
Linus Torvalds 已提交
2792 2793 2794 2795 2796 2797
	r->rtm_type	= rt->rt_type;
	r->rtm_scope	= RT_SCOPE_UNIVERSE;
	r->rtm_protocol = RTPROT_UNSPEC;
	r->rtm_flags	= (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
	if (rt->rt_flags & RTCF_NOTIFY)
		r->rtm_flags |= RTM_F_NOTIFY;
2798

2799
	NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
2800

2801
	if (rt->rt_key_src) {
L
Linus Torvalds 已提交
2802
		r->rtm_src_len = 32;
2803
		NLA_PUT_BE32(skb, RTA_SRC, rt->rt_key_src);
L
Linus Torvalds 已提交
2804
	}
2805 2806
	if (rt->dst.dev)
		NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
2807
#ifdef CONFIG_IP_ROUTE_CLASSID
2808 2809
	if (rt->dst.tclassid)
		NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
L
Linus Torvalds 已提交
2810
#endif
2811
	if (rt_is_input_route(rt))
2812
		NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
2813
	else if (rt->rt_src != rt->rt_key_src)
2814
		NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
2815

L
Linus Torvalds 已提交
2816
	if (rt->rt_dst != rt->rt_gateway)
2817
		NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
2818

2819
	if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
2820 2821
		goto nla_put_failure;

2822 2823
	if (rt->rt_mark)
		NLA_PUT_BE32(skb, RTA_MARK, rt->rt_mark);
E
Eric Dumazet 已提交
2824

2825
	error = rt->dst.error;
2826 2827
	expires = (rt->peer && rt->peer->pmtu_expires) ?
		rt->peer->pmtu_expires - jiffies : 0;
L
Linus Torvalds 已提交
2828
	if (rt->peer) {
2829
		inet_peer_refcheck(rt->peer);
E
Eric Dumazet 已提交
2830
		id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
L
Linus Torvalds 已提交
2831
		if (rt->peer->tcp_ts_stamp) {
2832
			ts = rt->peer->tcp_ts;
2833
			tsage = get_seconds() - rt->peer->tcp_ts_stamp;
L
Linus Torvalds 已提交
2834 2835
		}
	}
2836

2837
	if (rt_is_input_route(rt)) {
L
Linus Torvalds 已提交
2838
#ifdef CONFIG_IP_MROUTE
2839
		__be32 dst = rt->rt_dst;
L
Linus Torvalds 已提交
2840

2841
		if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2842
		    IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2843 2844 2845
			int err = ipmr_get_route(net, skb,
						 rt->rt_src, rt->rt_dst,
						 r, nowait);
L
Linus Torvalds 已提交
2846 2847 2848 2849
			if (err <= 0) {
				if (!nowait) {
					if (err == 0)
						return 0;
2850
					goto nla_put_failure;
L
Linus Torvalds 已提交
2851 2852
				} else {
					if (err == -EMSGSIZE)
2853
						goto nla_put_failure;
2854
					error = err;
L
Linus Torvalds 已提交
2855 2856 2857 2858
				}
			}
		} else
#endif
2859
			NLA_PUT_U32(skb, RTA_IIF, rt->rt_iif);
L
Linus Torvalds 已提交
2860 2861
	}

2862
	if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2863 2864
			       expires, error) < 0)
		goto nla_put_failure;
2865 2866

	return nlmsg_end(skb, nlh);
L
Linus Torvalds 已提交
2867

2868
nla_put_failure:
2869 2870
	nlmsg_cancel(skb, nlh);
	return -EMSGSIZE;
L
Linus Torvalds 已提交
2871 2872
}

2873
static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
L
Linus Torvalds 已提交
2874
{
2875
	struct net *net = sock_net(in_skb->sk);
2876 2877
	struct rtmsg *rtm;
	struct nlattr *tb[RTA_MAX+1];
L
Linus Torvalds 已提交
2878
	struct rtable *rt = NULL;
A
Al Viro 已提交
2879 2880 2881
	__be32 dst = 0;
	__be32 src = 0;
	u32 iif;
2882
	int err;
E
Eric Dumazet 已提交
2883
	int mark;
L
Linus Torvalds 已提交
2884 2885
	struct sk_buff *skb;

2886 2887 2888 2889 2890 2891
	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
	if (err < 0)
		goto errout;

	rtm = nlmsg_data(nlh);

L
Linus Torvalds 已提交
2892
	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2893 2894 2895 2896
	if (skb == NULL) {
		err = -ENOBUFS;
		goto errout;
	}
L
Linus Torvalds 已提交
2897 2898 2899 2900

	/* Reserve room for dummy headers, this skb can pass
	   through good chunk of routing engine.
	 */
2901
	skb_reset_mac_header(skb);
2902
	skb_reset_network_header(skb);
2903 2904

	/* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2905
	ip_hdr(skb)->protocol = IPPROTO_ICMP;
L
Linus Torvalds 已提交
2906 2907
	skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));

2908 2909
	src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
	dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2910
	iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
E
Eric Dumazet 已提交
2911
	mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
L
Linus Torvalds 已提交
2912 2913

	if (iif) {
2914 2915
		struct net_device *dev;

2916
		dev = __dev_get_by_index(net, iif);
2917 2918 2919 2920 2921
		if (dev == NULL) {
			err = -ENODEV;
			goto errout_free;
		}

L
Linus Torvalds 已提交
2922 2923
		skb->protocol	= htons(ETH_P_IP);
		skb->dev	= dev;
E
Eric Dumazet 已提交
2924
		skb->mark	= mark;
L
Linus Torvalds 已提交
2925 2926 2927
		local_bh_disable();
		err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
		local_bh_enable();
2928

E
Eric Dumazet 已提交
2929
		rt = skb_rtable(skb);
2930 2931
		if (err == 0 && rt->dst.error)
			err = -rt->dst.error;
L
Linus Torvalds 已提交
2932
	} else {
2933 2934 2935 2936 2937 2938
		struct flowi4 fl4 = {
			.daddr = dst,
			.saddr = src,
			.flowi4_tos = rtm->rtm_tos,
			.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
			.flowi4_mark = mark,
2939
		};
2940
		rt = ip_route_output_key(net, &fl4);
2941 2942 2943 2944

		err = 0;
		if (IS_ERR(rt))
			err = PTR_ERR(rt);
L
Linus Torvalds 已提交
2945
	}
2946

L
Linus Torvalds 已提交
2947
	if (err)
2948
		goto errout_free;
L
Linus Torvalds 已提交
2949

2950
	skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
2951 2952 2953
	if (rtm->rtm_flags & RTM_F_NOTIFY)
		rt->rt_flags |= RTCF_NOTIFY;

2954
	err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
2955
			   RTM_NEWROUTE, 0, 0);
2956 2957
	if (err <= 0)
		goto errout_free;
L
Linus Torvalds 已提交
2958

2959
	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
2960
errout:
2961
	return err;
L
Linus Torvalds 已提交
2962

2963
errout_free:
L
Linus Torvalds 已提交
2964
	kfree_skb(skb);
2965
	goto errout;
L
Linus Torvalds 已提交
2966 2967 2968 2969 2970 2971 2972
}

int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
{
	struct rtable *rt;
	int h, s_h;
	int idx, s_idx;
2973 2974
	struct net *net;

2975
	net = sock_net(skb->sk);
L
Linus Torvalds 已提交
2976 2977

	s_h = cb->args[0];
2978 2979
	if (s_h < 0)
		s_h = 0;
L
Linus Torvalds 已提交
2980
	s_idx = idx = cb->args[1];
2981 2982 2983
	for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
		if (!rt_hash_table[h].chain)
			continue;
L
Linus Torvalds 已提交
2984
		rcu_read_lock_bh();
2985
		for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
2986 2987
		     rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
			if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
L
Linus Torvalds 已提交
2988
				continue;
2989
			if (rt_is_expired(rt))
2990
				continue;
2991
			skb_dst_set_noref(skb, &rt->dst);
2992
			if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
2993
					 cb->nlh->nlmsg_seq, RTM_NEWROUTE,
2994
					 1, NLM_F_MULTI) <= 0) {
E
Eric Dumazet 已提交
2995
				skb_dst_drop(skb);
L
Linus Torvalds 已提交
2996 2997 2998
				rcu_read_unlock_bh();
				goto done;
			}
E
Eric Dumazet 已提交
2999
			skb_dst_drop(skb);
L
Linus Torvalds 已提交
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011
		}
		rcu_read_unlock_bh();
	}

done:
	cb->args[0] = h;
	cb->args[1] = idx;
	return skb->len;
}

void ip_rt_multicast_event(struct in_device *in_dev)
{
3012
	rt_cache_flush(dev_net(in_dev->dev), 0);
L
Linus Torvalds 已提交
3013 3014 3015
}

#ifdef CONFIG_SYSCTL
3016
static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3017
					void __user *buffer,
L
Linus Torvalds 已提交
3018 3019 3020
					size_t *lenp, loff_t *ppos)
{
	if (write) {
3021
		int flush_delay;
3022
		ctl_table ctl;
3023
		struct net *net;
3024

3025 3026
		memcpy(&ctl, __ctl, sizeof(ctl));
		ctl.data = &flush_delay;
3027
		proc_dointvec(&ctl, write, buffer, lenp, ppos);
3028

3029
		net = (struct net *)__ctl->extra1;
3030
		rt_cache_flush(net, flush_delay);
L
Linus Torvalds 已提交
3031
		return 0;
3032
	}
L
Linus Torvalds 已提交
3033 3034 3035 3036

	return -EINVAL;
}

A
Al Viro 已提交
3037
static ctl_table ipv4_route_table[] = {
L
Linus Torvalds 已提交
3038 3039 3040 3041 3042
	{
		.procname	= "gc_thresh",
		.data		= &ipv4_dst_ops.gc_thresh,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3043
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3044 3045 3046 3047 3048 3049
	},
	{
		.procname	= "max_size",
		.data		= &ip_rt_max_size,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3050
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3051 3052 3053
	},
	{
		/*  Deprecated. Use gc_min_interval_ms */
3054

L
Linus Torvalds 已提交
3055 3056 3057 3058
		.procname	= "gc_min_interval",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3059
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3060 3061 3062 3063 3064 3065
	},
	{
		.procname	= "gc_min_interval_ms",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3066
		.proc_handler	= proc_dointvec_ms_jiffies,
L
Linus Torvalds 已提交
3067 3068 3069 3070 3071 3072
	},
	{
		.procname	= "gc_timeout",
		.data		= &ip_rt_gc_timeout,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3073
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3074 3075 3076 3077 3078 3079
	},
	{
		.procname	= "gc_interval",
		.data		= &ip_rt_gc_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3080
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3081 3082 3083 3084 3085 3086
	},
	{
		.procname	= "redirect_load",
		.data		= &ip_rt_redirect_load,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3087
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3088 3089 3090 3091 3092 3093
	},
	{
		.procname	= "redirect_number",
		.data		= &ip_rt_redirect_number,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3094
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3095 3096 3097 3098 3099 3100
	},
	{
		.procname	= "redirect_silence",
		.data		= &ip_rt_redirect_silence,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3101
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3102 3103 3104 3105 3106 3107
	},
	{
		.procname	= "error_cost",
		.data		= &ip_rt_error_cost,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3108
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3109 3110 3111 3112 3113 3114
	},
	{
		.procname	= "error_burst",
		.data		= &ip_rt_error_burst,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3115
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3116 3117 3118 3119 3120 3121
	},
	{
		.procname	= "gc_elasticity",
		.data		= &ip_rt_gc_elasticity,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3122
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3123 3124 3125 3126 3127 3128
	},
	{
		.procname	= "mtu_expires",
		.data		= &ip_rt_mtu_expires,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3129
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3130 3131 3132 3133 3134 3135
	},
	{
		.procname	= "min_pmtu",
		.data		= &ip_rt_min_pmtu,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3136
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3137 3138 3139 3140 3141 3142
	},
	{
		.procname	= "min_adv_mss",
		.data		= &ip_rt_min_advmss,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3143
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3144
	},
3145
	{ }
L
Linus Torvalds 已提交
3146
};
3147

A
Al Viro 已提交
3148 3149 3150 3151
static struct ctl_table empty[1];

static struct ctl_table ipv4_skeleton[] =
{
3152
	{ .procname = "route", 
H
Hugh Dickins 已提交
3153
	  .mode = 0555, .child = ipv4_route_table},
3154
	{ .procname = "neigh", 
H
Hugh Dickins 已提交
3155
	  .mode = 0555, .child = empty},
A
Al Viro 已提交
3156 3157 3158 3159
	{ }
};

static __net_initdata struct ctl_path ipv4_path[] = {
3160 3161
	{ .procname = "net", },
	{ .procname = "ipv4", },
3162 3163 3164 3165 3166 3167 3168 3169
	{ },
};

static struct ctl_table ipv4_route_flush_table[] = {
	{
		.procname	= "flush",
		.maxlen		= sizeof(int),
		.mode		= 0200,
A
Alexey Dobriyan 已提交
3170
		.proc_handler	= ipv4_sysctl_rtcache_flush,
3171
	},
3172
	{ },
3173 3174
};

A
Al Viro 已提交
3175
static __net_initdata struct ctl_path ipv4_route_path[] = {
3176 3177 3178
	{ .procname = "net", },
	{ .procname = "ipv4", },
	{ .procname = "route", },
A
Al Viro 已提交
3179 3180 3181
	{ },
};

3182 3183 3184 3185 3186
static __net_init int sysctl_route_net_init(struct net *net)
{
	struct ctl_table *tbl;

	tbl = ipv4_route_flush_table;
O
Octavian Purdila 已提交
3187
	if (!net_eq(net, &init_net)) {
3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220
		tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
		if (tbl == NULL)
			goto err_dup;
	}
	tbl[0].extra1 = net;

	net->ipv4.route_hdr =
		register_net_sysctl_table(net, ipv4_route_path, tbl);
	if (net->ipv4.route_hdr == NULL)
		goto err_reg;
	return 0;

err_reg:
	if (tbl != ipv4_route_flush_table)
		kfree(tbl);
err_dup:
	return -ENOMEM;
}

static __net_exit void sysctl_route_net_exit(struct net *net)
{
	struct ctl_table *tbl;

	tbl = net->ipv4.route_hdr->ctl_table_arg;
	unregister_net_sysctl_table(net->ipv4.route_hdr);
	BUG_ON(tbl == ipv4_route_flush_table);
	kfree(tbl);
}

static __net_initdata struct pernet_operations sysctl_route_ops = {
	.init = sysctl_route_net_init,
	.exit = sysctl_route_net_exit,
};
L
Linus Torvalds 已提交
3221 3222
#endif

3223
static __net_init int rt_genid_init(struct net *net)
3224
{
3225 3226
	get_random_bytes(&net->ipv4.rt_genid,
			 sizeof(net->ipv4.rt_genid));
3227 3228
	get_random_bytes(&net->ipv4.dev_addr_genid,
			 sizeof(net->ipv4.dev_addr_genid));
3229 3230 3231
	return 0;
}

3232 3233
static __net_initdata struct pernet_operations rt_genid_ops = {
	.init = rt_genid_init,
3234 3235 3236
};


3237
#ifdef CONFIG_IP_ROUTE_CLASSID
3238
struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3239
#endif /* CONFIG_IP_ROUTE_CLASSID */
L
Linus Torvalds 已提交
3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252

static __initdata unsigned long rhash_entries;
static int __init set_rhash_entries(char *str)
{
	if (!str)
		return 0;
	rhash_entries = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("rhash_entries=", set_rhash_entries);

int __init ip_rt_init(void)
{
3253
	int rc = 0;
L
Linus Torvalds 已提交
3254

3255
#ifdef CONFIG_IP_ROUTE_CLASSID
3256
	ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
L
Linus Torvalds 已提交
3257 3258 3259 3260
	if (!ip_rt_acct)
		panic("IP: failed to allocate ip_rt_acct\n");
#endif

A
Alexey Dobriyan 已提交
3261 3262
	ipv4_dst_ops.kmem_cachep =
		kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3263
				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3264

3265 3266
	ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;

3267 3268 3269 3270 3271 3272
	if (dst_entries_init(&ipv4_dst_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_ops counter\n");

	if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");

3273 3274 3275 3276
	rt_hash_table = (struct rt_hash_bucket *)
		alloc_large_system_hash("IP route cache",
					sizeof(struct rt_hash_bucket),
					rhash_entries,
3277
					(totalram_pages >= 128 * 1024) ?
3278
					15 : 17,
3279
					0,
3280 3281
					&rt_hash_log,
					&rt_hash_mask,
3282
					rhash_entries ? 0 : 512 * 1024);
3283 3284
	memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
	rt_hash_lock_init();
L
Linus Torvalds 已提交
3285 3286 3287 3288 3289 3290 3291

	ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
	ip_rt_max_size = (rt_hash_mask + 1) * 16;

	devinet_init();
	ip_fib_init();

3292
	if (ip_rt_proc_init())
3293
		printk(KERN_ERR "Unable to create route proc files\n");
L
Linus Torvalds 已提交
3294 3295
#ifdef CONFIG_XFRM
	xfrm_init();
3296
	xfrm4_init(ip_rt_max_size);
L
Linus Torvalds 已提交
3297
#endif
3298 3299
	rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL);

3300 3301 3302
#ifdef CONFIG_SYSCTL
	register_pernet_subsys(&sysctl_route_ops);
#endif
3303
	register_pernet_subsys(&rt_genid_ops);
L
Linus Torvalds 已提交
3304 3305 3306
	return rc;
}

3307
#ifdef CONFIG_SYSCTL
A
Al Viro 已提交
3308 3309 3310 3311 3312 3313
/*
 * We really need to sanitize the damn ipv4 init order, then all
 * this nonsense will go away.
 */
void __init ip_static_sysctl_init(void)
{
A
Al Viro 已提交
3314
	register_sysctl_paths(ipv4_path, ipv4_skeleton);
A
Al Viro 已提交
3315
}
3316
#endif