route.c 82.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		ROUTE - implementation of the IP router.
 *
8
 * Authors:	Ross Biro
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
 *		Linus Torvalds, <Linus.Torvalds@helsinki.fi>
 *		Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 * Fixes:
 *		Alan Cox	:	Verify area fixes.
 *		Alan Cox	:	cli() protects routing changes
 *		Rui Oliveira	:	ICMP routing table updates
 *		(rco@di.uminho.pt)	Routing table insertion and update
 *		Linus Torvalds	:	Rewrote bits to be sensible
 *		Alan Cox	:	Added BSD route gw semantics
21
 *		Alan Cox	:	Super /proc >4K
L
Linus Torvalds 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
 *		Alan Cox	:	MTU in route table
 *		Alan Cox	: 	MSS actually. Also added the window
 *					clamper.
 *		Sam Lantinga	:	Fixed route matching in rt_del()
 *		Alan Cox	:	Routing cache support.
 *		Alan Cox	:	Removed compatibility cruft.
 *		Alan Cox	:	RTF_REJECT support.
 *		Alan Cox	:	TCP irtt support.
 *		Jonathan Naylor	:	Added Metric support.
 *	Miquel van Smoorenburg	:	BSD API fixes.
 *	Miquel van Smoorenburg	:	Metrics.
 *		Alan Cox	:	Use __u32 properly
 *		Alan Cox	:	Aligned routing errors more closely with BSD
 *					our system is still very different.
 *		Alan Cox	:	Faster /proc handling
 *	Alexey Kuznetsov	:	Massive rework to support tree based routing,
 *					routing caches and better behaviour.
39
 *
L
Linus Torvalds 已提交
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
 *		Olaf Erb	:	irtt wasn't being copied right.
 *		Bjorn Ekwall	:	Kerneld route support.
 *		Alan Cox	:	Multicast fixed (I hope)
 * 		Pavel Krauz	:	Limited broadcast fixed
 *		Mike McLagan	:	Routing by source
 *	Alexey Kuznetsov	:	End of old history. Split to fib.c and
 *					route.c and rewritten from scratch.
 *		Andi Kleen	:	Load-limit warning messages.
 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
 *	Vitaly E. Lavrov	:	Race condition in ip_route_input_slow.
 *	Tobias Ringstrom	:	Uninitialized res.type in ip_route_output_slow.
 *	Vladimir V. Ivanov	:	IP rule info (flowid) is really useful.
 *		Marc Boucher	:	routing by fwmark
 *	Robert Olsson		:	Added rt_cache statistics
 *	Arnaldo C. Melo		:	Convert proc stuff to seq_file
55
 *	Eric Dumazet		:	hashed spinlocks and rt_check_expire() fixes.
56 57
 * 	Ilia Sotnikov		:	Ignore TOS on PMTUD and Redirect
 * 	Ilia Sotnikov		:	Removed TOS from hash calculations
L
Linus Torvalds 已提交
58 59 60 61 62 63 64 65 66 67 68 69 70 71
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 */

#include <linux/module.h>
#include <asm/uaccess.h>
#include <asm/system.h>
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
72
#include <linux/bootmem.h>
L
Linus Torvalds 已提交
73 74 75 76 77 78 79 80 81
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
82
#include <linux/workqueue.h>
L
Linus Torvalds 已提交
83 84 85 86 87 88 89 90 91 92
#include <linux/skbuff.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include <linux/pkt_sched.h>
#include <linux/mroute.h>
#include <linux/netfilter_ipv4.h>
#include <linux/random.h>
#include <linux/jhash.h>
#include <linux/rcupdate.h>
#include <linux/times.h>
93
#include <linux/slab.h>
94
#include <net/dst.h>
95
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104 105
#include <net/protocol.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/inetpeer.h>
#include <net/sock.h>
#include <net/ip_fib.h>
#include <net/arp.h>
#include <net/tcp.h>
#include <net/icmp.h>
#include <net/xfrm.h>
106
#include <net/netevent.h>
107
#include <net/rtnetlink.h>
L
Linus Torvalds 已提交
108 109 110 111 112 113 114 115 116 117 118 119
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif

#define RT_FL_TOS(oldflp) \
    ((u32)(oldflp->fl4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))

#define IP_MAX_MTU	0xFFF0

#define RT_GC_TIMEOUT (300*HZ)

static int ip_rt_max_size;
120 121 122 123 124 125 126 127 128 129 130 131
static int ip_rt_gc_timeout __read_mostly	= RT_GC_TIMEOUT;
static int ip_rt_gc_interval __read_mostly	= 60 * HZ;
static int ip_rt_gc_min_interval __read_mostly	= HZ / 2;
static int ip_rt_redirect_number __read_mostly	= 9;
static int ip_rt_redirect_load __read_mostly	= HZ / 50;
static int ip_rt_redirect_silence __read_mostly	= ((HZ / 50) << (9 + 1));
static int ip_rt_error_cost __read_mostly	= HZ;
static int ip_rt_error_burst __read_mostly	= 5 * HZ;
static int ip_rt_gc_elasticity __read_mostly	= 8;
static int ip_rt_mtu_expires __read_mostly	= 10 * 60 * HZ;
static int ip_rt_min_pmtu __read_mostly		= 512 + 20 + 20;
static int ip_rt_min_advmss __read_mostly	= 256;
132
static int rt_chain_length_max __read_mostly	= 20;
L
Linus Torvalds 已提交
133

134 135
static struct delayed_work expires_work;
static unsigned long expires_ljiffies;
L
Linus Torvalds 已提交
136 137 138 139 140 141 142 143 144 145

/*
 *	Interface to generic destination cache.
 */

static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
static void		 ipv4_dst_destroy(struct dst_entry *dst);
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
static void		 ipv4_link_failure(struct sk_buff *skb);
static void		 ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
146
static int rt_garbage_collect(struct dst_ops *ops);
L
Linus Torvalds 已提交
147

E
Eric Dumazet 已提交
148 149 150 151
static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
			    int how)
{
}
L
Linus Torvalds 已提交
152 153 154

static struct dst_ops ipv4_dst_ops = {
	.family =		AF_INET,
155
	.protocol =		cpu_to_be16(ETH_P_IP),
L
Linus Torvalds 已提交
156 157 158 159 160 161 162
	.gc =			rt_garbage_collect,
	.check =		ipv4_dst_check,
	.destroy =		ipv4_dst_destroy,
	.ifdown =		ipv4_dst_ifdown,
	.negative_advice =	ipv4_negative_advice,
	.link_failure =		ipv4_link_failure,
	.update_pmtu =		ip_rt_update_pmtu,
163
	.local_out =		__ip_local_out,
L
Linus Torvalds 已提交
164 165 166 167
};

#define ECN_OR_COST(class)	TC_PRIO_##class

168
const __u8 ip_tos2prio[16] = {
L
Linus Torvalds 已提交
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
	TC_PRIO_BESTEFFORT,
	ECN_OR_COST(FILLER),
	TC_PRIO_BESTEFFORT,
	ECN_OR_COST(BESTEFFORT),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_BULK,
	ECN_OR_COST(BULK),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE,
	ECN_OR_COST(INTERACTIVE),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK),
	TC_PRIO_INTERACTIVE_BULK,
	ECN_OR_COST(INTERACTIVE_BULK)
};


/*
 * Route cache.
 */

/* The locking scheme is rather straight forward:
 *
 * 1) Read-Copy Update protects the buckets of the central route hash.
 * 2) Only writers remove entries, and they hold the lock
 *    as they look at rtable reference counts.
 * 3) Only readers acquire references to rtable entries,
 *    they do so with atomic increments and with the
 *    lock held.
 */

struct rt_hash_bucket {
203
	struct rtable __rcu	*chain;
204
};
205

206 207
#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
	defined(CONFIG_PROVE_LOCKING)
208 209 210
/*
 * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
 * The size of this table is a power of two and depends on the number of CPUS.
211
 * (on lockdep we have a quite big spinlock_t, so keep the size down there)
212
 */
213 214
#ifdef CONFIG_LOCKDEP
# define RT_HASH_LOCK_SZ	256
215
#else
216 217 218 219 220 221 222 223 224 225 226
# if NR_CPUS >= 32
#  define RT_HASH_LOCK_SZ	4096
# elif NR_CPUS >= 16
#  define RT_HASH_LOCK_SZ	2048
# elif NR_CPUS >= 8
#  define RT_HASH_LOCK_SZ	1024
# elif NR_CPUS >= 4
#  define RT_HASH_LOCK_SZ	512
# else
#  define RT_HASH_LOCK_SZ	256
# endif
227 228 229 230
#endif

static spinlock_t	*rt_hash_locks;
# define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
231 232 233 234 235 236 237 238 239 240 241 242 243

static __init void rt_hash_lock_init(void)
{
	int i;

	rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
			GFP_KERNEL);
	if (!rt_hash_locks)
		panic("IP: failed to allocate rt_hash_locks\n");

	for (i = 0; i < RT_HASH_LOCK_SZ; i++)
		spin_lock_init(&rt_hash_locks[i]);
}
244 245
#else
# define rt_hash_lock_addr(slot) NULL
246 247 248 249

static inline void rt_hash_lock_init(void)
{
}
250
#endif
L
Linus Torvalds 已提交
251

252 253 254
static struct rt_hash_bucket 	*rt_hash_table __read_mostly;
static unsigned			rt_hash_mask __read_mostly;
static unsigned int		rt_hash_log  __read_mostly;
L
Linus Torvalds 已提交
255

256
static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
257
#define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
L
Linus Torvalds 已提交
258

259
static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
260
				   int genid)
L
Linus Torvalds 已提交
261
{
262
	return jhash_3words((__force u32)daddr, (__force u32)saddr,
263
			    idx, genid)
264
		& rt_hash_mask;
L
Linus Torvalds 已提交
265 266
}

267 268 269 270 271
static inline int rt_genid(struct net *net)
{
	return atomic_read(&net->ipv4.rt_genid);
}

L
Linus Torvalds 已提交
272 273
#ifdef CONFIG_PROC_FS
struct rt_cache_iter_state {
274
	struct seq_net_private p;
L
Linus Torvalds 已提交
275
	int bucket;
276
	int genid;
L
Linus Torvalds 已提交
277 278
};

279
static struct rtable *rt_cache_get_first(struct seq_file *seq)
L
Linus Torvalds 已提交
280
{
281
	struct rt_cache_iter_state *st = seq->private;
L
Linus Torvalds 已提交
282 283 284
	struct rtable *r = NULL;

	for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
285
		if (!rcu_dereference_raw(rt_hash_table[st->bucket].chain))
286
			continue;
L
Linus Torvalds 已提交
287
		rcu_read_lock_bh();
288
		r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
289
		while (r) {
290
			if (dev_net(r->dst.dev) == seq_file_net(seq) &&
291
			    r->rt_genid == st->genid)
292
				return r;
293
			r = rcu_dereference_bh(r->dst.rt_next);
294
		}
L
Linus Torvalds 已提交
295 296
		rcu_read_unlock_bh();
	}
297
	return r;
L
Linus Torvalds 已提交
298 299
}

300
static struct rtable *__rt_cache_get_next(struct seq_file *seq,
301
					  struct rtable *r)
L
Linus Torvalds 已提交
302
{
303
	struct rt_cache_iter_state *st = seq->private;
304

305
	r = rcu_dereference_bh(r->dst.rt_next);
L
Linus Torvalds 已提交
306 307
	while (!r) {
		rcu_read_unlock_bh();
308 309 310
		do {
			if (--st->bucket < 0)
				return NULL;
311
		} while (!rcu_dereference_raw(rt_hash_table[st->bucket].chain));
L
Linus Torvalds 已提交
312
		rcu_read_lock_bh();
313
		r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
L
Linus Torvalds 已提交
314
	}
315
	return r;
L
Linus Torvalds 已提交
316 317
}

318
static struct rtable *rt_cache_get_next(struct seq_file *seq,
319 320
					struct rtable *r)
{
321 322
	struct rt_cache_iter_state *st = seq->private;
	while ((r = __rt_cache_get_next(seq, r)) != NULL) {
323
		if (dev_net(r->dst.dev) != seq_file_net(seq))
324
			continue;
325 326 327 328 329 330
		if (r->rt_genid == st->genid)
			break;
	}
	return r;
}

331
static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
L
Linus Torvalds 已提交
332
{
333
	struct rtable *r = rt_cache_get_first(seq);
L
Linus Torvalds 已提交
334 335

	if (r)
336
		while (pos && (r = rt_cache_get_next(seq, r)))
L
Linus Torvalds 已提交
337 338 339 340 341 342
			--pos;
	return pos ? NULL : r;
}

static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
{
343 344
	struct rt_cache_iter_state *st = seq->private;
	if (*pos)
345
		return rt_cache_get_idx(seq, *pos - 1);
346
	st->genid = rt_genid(seq_file_net(seq));
347
	return SEQ_START_TOKEN;
L
Linus Torvalds 已提交
348 349 350 351
}

static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
352
	struct rtable *r;
L
Linus Torvalds 已提交
353 354

	if (v == SEQ_START_TOKEN)
355
		r = rt_cache_get_first(seq);
L
Linus Torvalds 已提交
356
	else
357
		r = rt_cache_get_next(seq, v);
L
Linus Torvalds 已提交
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
	++*pos;
	return r;
}

static void rt_cache_seq_stop(struct seq_file *seq, void *v)
{
	if (v && v != SEQ_START_TOKEN)
		rcu_read_unlock_bh();
}

static int rt_cache_seq_show(struct seq_file *seq, void *v)
{
	if (v == SEQ_START_TOKEN)
		seq_printf(seq, "%-127s\n",
			   "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
			   "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
			   "HHUptod\tSpecDst");
	else {
		struct rtable *r = v;
377
		int len;
L
Linus Torvalds 已提交
378

379 380
		seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
			      "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
381
			r->dst.dev ? r->dst.dev->name : "*",
382 383
			(__force u32)r->rt_dst,
			(__force u32)r->rt_gateway,
384 385 386 387 388 389 390
			r->rt_flags, atomic_read(&r->dst.__refcnt),
			r->dst.__use, 0, (__force u32)r->rt_src,
			(dst_metric(&r->dst, RTAX_ADVMSS) ?
			     (int)dst_metric(&r->dst, RTAX_ADVMSS) + 40 : 0),
			dst_metric(&r->dst, RTAX_WINDOW),
			(int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
			      dst_metric(&r->dst, RTAX_RTTVAR)),
L
Linus Torvalds 已提交
391
			r->fl.fl4_tos,
392 393
			r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
			r->dst.hh ? (r->dst.hh->hh_output ==
L
Linus Torvalds 已提交
394
				       dev_queue_xmit) : 0,
395 396 397
			r->rt_spec_dst, &len);

		seq_printf(seq, "%*s\n", 127 - len, "");
398 399
	}
	return 0;
L
Linus Torvalds 已提交
400 401
}

402
static const struct seq_operations rt_cache_seq_ops = {
L
Linus Torvalds 已提交
403 404 405 406 407 408 409 410
	.start  = rt_cache_seq_start,
	.next   = rt_cache_seq_next,
	.stop   = rt_cache_seq_stop,
	.show   = rt_cache_seq_show,
};

static int rt_cache_seq_open(struct inode *inode, struct file *file)
{
411
	return seq_open_net(inode, file, &rt_cache_seq_ops,
412
			sizeof(struct rt_cache_iter_state));
L
Linus Torvalds 已提交
413 414
}

415
static const struct file_operations rt_cache_seq_fops = {
L
Linus Torvalds 已提交
416 417 418 419
	.owner	 = THIS_MODULE,
	.open	 = rt_cache_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
420
	.release = seq_release_net,
L
Linus Torvalds 已提交
421 422 423 424 425 426 427 428 429 430
};


static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
{
	int cpu;

	if (*pos == 0)
		return SEQ_START_TOKEN;

431
	for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
432 433 434
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
435
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
436 437 438 439 440 441 442 443
	}
	return NULL;
}

static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	int cpu;

444
	for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
L
Linus Torvalds 已提交
445 446 447
		if (!cpu_possible(cpu))
			continue;
		*pos = cpu+1;
448
		return &per_cpu(rt_cache_stat, cpu);
L
Linus Torvalds 已提交
449 450
	}
	return NULL;
451

L
Linus Torvalds 已提交
452 453 454 455 456 457 458 459 460 461 462 463
}

static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
{

}

static int rt_cpu_seq_show(struct seq_file *seq, void *v)
{
	struct rt_cache_stat *st = v;

	if (v == SEQ_START_TOKEN) {
464
		seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
L
Linus Torvalds 已提交
465 466
		return 0;
	}
467

L
Linus Torvalds 已提交
468 469
	seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
		   " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
470
		   dst_entries_get_slow(&ipv4_dst_ops),
L
Linus Torvalds 已提交
471 472 473 474 475 476 477 478 479 480
		   st->in_hit,
		   st->in_slow_tot,
		   st->in_slow_mc,
		   st->in_no_route,
		   st->in_brd,
		   st->in_martian_dst,
		   st->in_martian_src,

		   st->out_hit,
		   st->out_slow_tot,
481
		   st->out_slow_mc,
L
Linus Torvalds 已提交
482 483 484 485 486 487 488 489 490 491 492

		   st->gc_total,
		   st->gc_ignored,
		   st->gc_goal_miss,
		   st->gc_dst_overflow,
		   st->in_hlist_search,
		   st->out_hlist_search
		);
	return 0;
}

493
static const struct seq_operations rt_cpu_seq_ops = {
L
Linus Torvalds 已提交
494 495 496 497 498 499 500 501 502 503 504 505
	.start  = rt_cpu_seq_start,
	.next   = rt_cpu_seq_next,
	.stop   = rt_cpu_seq_stop,
	.show   = rt_cpu_seq_show,
};


static int rt_cpu_seq_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &rt_cpu_seq_ops);
}

506
static const struct file_operations rt_cpu_seq_fops = {
L
Linus Torvalds 已提交
507 508 509 510 511 512 513
	.owner	 = THIS_MODULE,
	.open	 = rt_cpu_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
	.release = seq_release,
};

514
#ifdef CONFIG_NET_CLS_ROUTE
515
static int rt_acct_proc_show(struct seq_file *m, void *v)
516
{
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531
	struct ip_rt_acct *dst, *src;
	unsigned int i, j;

	dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
	if (!dst)
		return -ENOMEM;

	for_each_possible_cpu(i) {
		src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
		for (j = 0; j < 256; j++) {
			dst[j].o_bytes   += src[j].o_bytes;
			dst[j].o_packets += src[j].o_packets;
			dst[j].i_bytes   += src[j].i_bytes;
			dst[j].i_packets += src[j].i_packets;
		}
532 533
	}

534 535 536 537
	seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
	kfree(dst);
	return 0;
}
538

539 540 541
static int rt_acct_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, rt_acct_proc_show, NULL);
542
}
543 544 545 546 547 548 549 550

static const struct file_operations rt_acct_proc_fops = {
	.owner		= THIS_MODULE,
	.open		= rt_acct_proc_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};
551
#endif
552

553
static int __net_init ip_rt_do_proc_init(struct net *net)
554 555 556 557 558 559 560 561
{
	struct proc_dir_entry *pde;

	pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
			&rt_cache_seq_fops);
	if (!pde)
		goto err1;

562 563
	pde = proc_create("rt_cache", S_IRUGO,
			  net->proc_net_stat, &rt_cpu_seq_fops);
564 565 566 567
	if (!pde)
		goto err2;

#ifdef CONFIG_NET_CLS_ROUTE
568
	pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
569 570 571 572 573 574 575 576 577 578 579 580 581 582
	if (!pde)
		goto err3;
#endif
	return 0;

#ifdef CONFIG_NET_CLS_ROUTE
err3:
	remove_proc_entry("rt_cache", net->proc_net_stat);
#endif
err2:
	remove_proc_entry("rt_cache", net->proc_net);
err1:
	return -ENOMEM;
}
583 584 585 586 587

static void __net_exit ip_rt_do_proc_exit(struct net *net)
{
	remove_proc_entry("rt_cache", net->proc_net_stat);
	remove_proc_entry("rt_cache", net->proc_net);
588
#ifdef CONFIG_NET_CLS_ROUTE
589
	remove_proc_entry("rt_acct", net->proc_net);
590
#endif
591 592 593 594 595 596 597 598 599 600 601 602
}

static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
	.init = ip_rt_do_proc_init,
	.exit = ip_rt_do_proc_exit,
};

static int __init ip_rt_proc_init(void)
{
	return register_pernet_subsys(&ip_rt_proc_ops);
}

603
#else
604
static inline int ip_rt_proc_init(void)
605 606 607
{
	return 0;
}
L
Linus Torvalds 已提交
608
#endif /* CONFIG_PROC_FS */
609

S
Stephen Hemminger 已提交
610
static inline void rt_free(struct rtable *rt)
L
Linus Torvalds 已提交
611
{
612
	call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
L
Linus Torvalds 已提交
613 614
}

S
Stephen Hemminger 已提交
615
static inline void rt_drop(struct rtable *rt)
L
Linus Torvalds 已提交
616 617
{
	ip_rt_put(rt);
618
	call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
L
Linus Torvalds 已提交
619 620
}

S
Stephen Hemminger 已提交
621
static inline int rt_fast_clean(struct rtable *rth)
L
Linus Torvalds 已提交
622 623 624 625
{
	/* Kill broadcast/multicast entries very aggresively, if they
	   collide in hash table with more useful entries */
	return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
626
		rt_is_input_route(rth) && rth->dst.rt_next;
L
Linus Torvalds 已提交
627 628
}

S
Stephen Hemminger 已提交
629
static inline int rt_valuable(struct rtable *rth)
L
Linus Torvalds 已提交
630 631
{
	return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
632
		rth->dst.expires;
L
Linus Torvalds 已提交
633 634 635 636 637 638 639
}

static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
{
	unsigned long age;
	int ret = 0;

640
	if (atomic_read(&rth->dst.__refcnt))
L
Linus Torvalds 已提交
641 642 643
		goto out;

	ret = 1;
644 645
	if (rth->dst.expires &&
	    time_after_eq(jiffies, rth->dst.expires))
L
Linus Torvalds 已提交
646 647
		goto out;

648
	age = jiffies - rth->dst.lastuse;
L
Linus Torvalds 已提交
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663
	ret = 0;
	if ((age <= tmo1 && !rt_fast_clean(rth)) ||
	    (age <= tmo2 && rt_valuable(rth)))
		goto out;
	ret = 1;
out:	return ret;
}

/* Bits of score are:
 * 31: very valuable
 * 30: not quite useless
 * 29..0: usage counter
 */
static inline u32 rt_score(struct rtable *rt)
{
664
	u32 score = jiffies - rt->dst.lastuse;
L
Linus Torvalds 已提交
665 666 667 668 669 670

	score = ~score & ~(3<<30);

	if (rt_valuable(rt))
		score |= (1<<31);

671
	if (rt_is_output_route(rt) ||
L
Linus Torvalds 已提交
672 673 674 675 676 677
	    !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
		score |= (1<<30);

	return score;
}

678 679 680 681 682 683 684 685 686
static inline bool rt_caching(const struct net *net)
{
	return net->ipv4.current_rt_cache_rebuild_count <=
		net->ipv4.sysctl_rt_cache_rebuild_count;
}

static inline bool compare_hash_inputs(const struct flowi *fl1,
					const struct flowi *fl2)
{
687 688
	return ((((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) |
		((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) |
689 690 691
		(fl1->iif ^ fl2->iif)) == 0);
}

L
Linus Torvalds 已提交
692 693
static inline int compare_keys(struct flowi *fl1, struct flowi *fl2)
{
694 695
	return (((__force u32)fl1->fl4_dst ^ (__force u32)fl2->fl4_dst) |
		((__force u32)fl1->fl4_src ^ (__force u32)fl2->fl4_src) |
696
		(fl1->mark ^ fl2->mark) |
697
		(*(u16 *)&fl1->fl4_tos ^ *(u16 *)&fl2->fl4_tos) |
698 699
		(fl1->oif ^ fl2->oif) |
		(fl1->iif ^ fl2->iif)) == 0;
L
Linus Torvalds 已提交
700 701
}

702 703
static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
{
704
	return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
705 706
}

707 708
static inline int rt_is_expired(struct rtable *rth)
{
709
	return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
710 711
}

712 713 714 715 716 717 718 719 720
/*
 * Perform a full scan of hash table and free all entries.
 * Can be called by a softirq or a process.
 * In the later case, we want to be reschedule if necessary
 */
static void rt_do_flush(int process_context)
{
	unsigned int i;
	struct rtable *rth, *next;
721
	struct rtable * tail;
722 723 724 725

	for (i = 0; i <= rt_hash_mask; i++) {
		if (process_context && need_resched())
			cond_resched();
726
		rth = rcu_dereference_raw(rt_hash_table[i].chain);
727 728 729 730
		if (!rth)
			continue;

		spin_lock_bh(rt_hash_lock_addr(i));
731 732
#ifdef CONFIG_NET_NS
		{
733 734
		struct rtable __rcu **prev;
		struct rtable *p;
735

736 737
		rth = rcu_dereference_protected(rt_hash_table[i].chain,
			lockdep_is_held(rt_hash_lock_addr(i)));
738 739

		/* defer releasing the head of the list after spin_unlock */
740 741 742
		for (tail = rth; tail;
		     tail = rcu_dereference_protected(tail->dst.rt_next,
				lockdep_is_held(rt_hash_lock_addr(i))))
743 744 745 746 747 748 749
			if (!rt_is_expired(tail))
				break;
		if (rth != tail)
			rt_hash_table[i].chain = tail;

		/* call rt_free on entries after the tail requiring flush */
		prev = &rt_hash_table[i].chain;
750 751 752 753 754 755
		for (p = rcu_dereference_protected(*prev,
				lockdep_is_held(rt_hash_lock_addr(i)));
		     p != NULL;
		     p = next) {
			next = rcu_dereference_protected(p->dst.rt_next,
				lockdep_is_held(rt_hash_lock_addr(i)));
756
			if (!rt_is_expired(p)) {
757
				prev = &p->dst.rt_next;
758 759 760 761 762 763 764
			} else {
				*prev = next;
				rt_free(p);
			}
		}
		}
#else
765 766 767
		rth = rcu_dereference_protected(rt_hash_table[i].chain,
			lockdep_is_held(rt_hash_lock_addr(i)));
		rcu_assign_pointer(rt_hash_table[i].chain, NULL);
768 769
		tail = NULL;
#endif
770 771
		spin_unlock_bh(rt_hash_lock_addr(i));

772
		for (; rth != tail; rth = next) {
773
			next = rcu_dereference_protected(rth->dst.rt_next, 1);
774 775 776 777 778
			rt_free(rth);
		}
	}
}

779 780 781 782 783 784 785 786 787 788 789
/*
 * While freeing expired entries, we compute average chain length
 * and standard deviation, using fixed-point arithmetic.
 * This to have an estimation of rt_chain_length_max
 *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
 * We use 3 bits for frational part, and 29 (or 61) for magnitude.
 */

#define FRACT_BITS 3
#define ONE (1UL << FRACT_BITS)

E
Eric Dumazet 已提交
790 791 792 793 794 795 796 797 798 799 800 801 802 803
/*
 * Given a hash chain and an item in this hash chain,
 * find if a previous entry has the same hash_inputs
 * (but differs on tos, mark or oif)
 * Returns 0 if an alias is found.
 * Returns ONE if rth has no alias before itself.
 */
static int has_noalias(const struct rtable *head, const struct rtable *rth)
{
	const struct rtable *aux = head;

	while (aux != rth) {
		if (compare_hash_inputs(&aux->fl, &rth->fl))
			return 0;
804
		aux = rcu_dereference_protected(aux->dst.rt_next, 1);
E
Eric Dumazet 已提交
805 806 807 808
	}
	return ONE;
}

809
static void rt_check_expire(void)
L
Linus Torvalds 已提交
810
{
811 812
	static unsigned int rover;
	unsigned int i = rover, goal;
813 814
	struct rtable *rth;
	struct rtable __rcu **rthp;
815
	unsigned long samples = 0;
816
	unsigned long sum = 0, sum2 = 0;
817
	unsigned long delta;
818 819
	u64 mult;

820 821 822
	delta = jiffies - expires_ljiffies;
	expires_ljiffies = jiffies;
	mult = ((u64)delta) << rt_hash_log;
823 824 825
	if (ip_rt_gc_timeout > 1)
		do_div(mult, ip_rt_gc_timeout);
	goal = (unsigned int)mult;
826 827
	if (goal > rt_hash_mask)
		goal = rt_hash_mask + 1;
828
	for (; goal > 0; goal--) {
L
Linus Torvalds 已提交
829
		unsigned long tmo = ip_rt_gc_timeout;
830
		unsigned long length;
L
Linus Torvalds 已提交
831 832 833 834

		i = (i + 1) & rt_hash_mask;
		rthp = &rt_hash_table[i].chain;

835 836 837
		if (need_resched())
			cond_resched();

838 839
		samples++;

840
		if (rcu_dereference_raw(*rthp) == NULL)
841
			continue;
842
		length = 0;
843
		spin_lock_bh(rt_hash_lock_addr(i));
844 845
		while ((rth = rcu_dereference_protected(*rthp,
					lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) {
846
			prefetch(rth->dst.rt_next);
847
			if (rt_is_expired(rth)) {
848
				*rthp = rth->dst.rt_next;
849 850 851
				rt_free(rth);
				continue;
			}
852
			if (rth->dst.expires) {
L
Linus Torvalds 已提交
853
				/* Entry is expired even if it is in use */
854
				if (time_before_eq(jiffies, rth->dst.expires)) {
855
nofree:
L
Linus Torvalds 已提交
856
					tmo >>= 1;
857
					rthp = &rth->dst.rt_next;
858
					/*
859
					 * We only count entries on
860 861 862 863 864 865
					 * a chain with equal hash inputs once
					 * so that entries for different QOS
					 * levels, and other non-hash input
					 * attributes don't unfairly skew
					 * the length computation
					 */
E
Eric Dumazet 已提交
866
					length += has_noalias(rt_hash_table[i].chain, rth);
L
Linus Torvalds 已提交
867 868
					continue;
				}
869 870
			} else if (!rt_may_expire(rth, tmo, ip_rt_gc_timeout))
				goto nofree;
L
Linus Torvalds 已提交
871 872

			/* Cleanup aged off entries. */
873
			*rthp = rth->dst.rt_next;
874
			rt_free(rth);
L
Linus Torvalds 已提交
875
		}
876
		spin_unlock_bh(rt_hash_lock_addr(i));
877 878 879 880 881 882 883 884 885
		sum += length;
		sum2 += length*length;
	}
	if (samples) {
		unsigned long avg = sum / samples;
		unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
		rt_chain_length_max = max_t(unsigned long,
					ip_rt_gc_elasticity,
					(avg + 4*sd) >> FRACT_BITS);
L
Linus Torvalds 已提交
886 887
	}
	rover = i;
888 889 890 891
}

/*
 * rt_worker_func() is run in process context.
892
 * we call rt_check_expire() to scan part of the hash table
893 894 895
 */
static void rt_worker_func(struct work_struct *work)
{
896
	rt_check_expire();
897
	schedule_delayed_work(&expires_work, ip_rt_gc_interval);
L
Linus Torvalds 已提交
898 899
}

900 901 902 903 904
/*
 * Pertubation of rt_genid by a small quantity [1..256]
 * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
 * many times (2^24) without giving recent rt_genid.
 * Jenkins hash is strong enough that litle changes of rt_genid are OK.
L
Linus Torvalds 已提交
905
 */
906
static void rt_cache_invalidate(struct net *net)
L
Linus Torvalds 已提交
907
{
908
	unsigned char shuffle;
L
Linus Torvalds 已提交
909

910
	get_random_bytes(&shuffle, sizeof(shuffle));
911
	atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
L
Linus Torvalds 已提交
912 913
}

914 915 916 917
/*
 * delay < 0  : invalidate cache (fast : entries will be deleted later)
 * delay >= 0 : invalidate & flush cache (can be long)
 */
918
void rt_cache_flush(struct net *net, int delay)
L
Linus Torvalds 已提交
919
{
920
	rt_cache_invalidate(net);
921 922
	if (delay >= 0)
		rt_do_flush(!in_softirq());
L
Linus Torvalds 已提交
923 924
}

925 926 927 928 929 930
/* Flush previous cache invalidated entries from the cache */
void rt_cache_flush_batch(void)
{
	rt_do_flush(!in_softirq());
}

931 932
static void rt_emergency_hash_rebuild(struct net *net)
{
933
	if (net_ratelimit())
934
		printk(KERN_WARNING "Route hash chain too long!\n");
935
	rt_cache_invalidate(net);
936 937
}

L
Linus Torvalds 已提交
938 939 940 941 942 943 944 945 946 947 948 949 950
/*
   Short description of GC goals.

   We want to build algorithm, which will keep routing cache
   at some equilibrium point, when number of aged off entries
   is kept approximately equal to newly generated ones.

   Current expiration strength is variable "expire".
   We try to adjust it dynamically, so that if networking
   is idle expires is large enough to keep enough of warm entries,
   and when load increases it reduces to limit cache size.
 */

951
static int rt_garbage_collect(struct dst_ops *ops)
L
Linus Torvalds 已提交
952 953 954 955 956
{
	static unsigned long expire = RT_GC_TIMEOUT;
	static unsigned long last_gc;
	static int rover;
	static int equilibrium;
957 958
	struct rtable *rth;
	struct rtable __rcu **rthp;
L
Linus Torvalds 已提交
959 960
	unsigned long now = jiffies;
	int goal;
961
	int entries = dst_entries_get_fast(&ipv4_dst_ops);
L
Linus Torvalds 已提交
962 963 964 965 966 967 968 969 970

	/*
	 * Garbage collection is pretty expensive,
	 * do not make it too frequently.
	 */

	RT_CACHE_STAT_INC(gc_total);

	if (now - last_gc < ip_rt_gc_min_interval &&
971
	    entries < ip_rt_max_size) {
L
Linus Torvalds 已提交
972 973 974 975
		RT_CACHE_STAT_INC(gc_ignored);
		goto out;
	}

976
	entries = dst_entries_get_slow(&ipv4_dst_ops);
L
Linus Torvalds 已提交
977
	/* Calculate number of entries, which we want to expire now. */
978
	goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
L
Linus Torvalds 已提交
979 980 981
	if (goal <= 0) {
		if (equilibrium < ipv4_dst_ops.gc_thresh)
			equilibrium = ipv4_dst_ops.gc_thresh;
982
		goal = entries - equilibrium;
L
Linus Torvalds 已提交
983
		if (goal > 0) {
984
			equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
985
			goal = entries - equilibrium;
L
Linus Torvalds 已提交
986 987 988 989 990
		}
	} else {
		/* We are in dangerous area. Try to reduce cache really
		 * aggressively.
		 */
991
		goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
992
		equilibrium = entries - goal;
L
Linus Torvalds 已提交
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
	}

	if (now - last_gc >= ip_rt_gc_min_interval)
		last_gc = now;

	if (goal <= 0) {
		equilibrium += goal;
		goto work_done;
	}

	do {
		int i, k;

		for (i = rt_hash_mask, k = rover; i >= 0; i--) {
			unsigned long tmo = expire;

			k = (k + 1) & rt_hash_mask;
			rthp = &rt_hash_table[k].chain;
1011
			spin_lock_bh(rt_hash_lock_addr(k));
1012 1013
			while ((rth = rcu_dereference_protected(*rthp,
					lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
1014
				if (!rt_is_expired(rth) &&
1015
					!rt_may_expire(rth, tmo, expire)) {
L
Linus Torvalds 已提交
1016
					tmo >>= 1;
1017
					rthp = &rth->dst.rt_next;
L
Linus Torvalds 已提交
1018 1019
					continue;
				}
1020
				*rthp = rth->dst.rt_next;
L
Linus Torvalds 已提交
1021 1022 1023
				rt_free(rth);
				goal--;
			}
1024
			spin_unlock_bh(rt_hash_lock_addr(k));
L
Linus Torvalds 已提交
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
			if (goal <= 0)
				break;
		}
		rover = k;

		if (goal <= 0)
			goto work_done;

		/* Goal is not achieved. We stop process if:

		   - if expire reduced to zero. Otherwise, expire is halfed.
		   - if table is not full.
		   - if we are called from interrupt.
		   - jiffies check is just fallback/debug loop breaker.
		     We will not spin here for long time in any case.
		 */

		RT_CACHE_STAT_INC(gc_goal_miss);

		if (expire == 0)
			break;

		expire >>= 1;
#if RT_CACHE_DEBUG >= 2
		printk(KERN_DEBUG "expire>> %u %d %d %d\n", expire,
1050
				dst_entries_get_fast(&ipv4_dst_ops), goal, i);
L
Linus Torvalds 已提交
1051 1052
#endif

1053
		if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
L
Linus Torvalds 已提交
1054 1055 1056
			goto out;
	} while (!in_softirq() && time_before_eq(jiffies, now));

1057 1058 1059
	if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
		goto out;
	if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
L
Linus Torvalds 已提交
1060 1061 1062 1063 1064 1065 1066 1067 1068
		goto out;
	if (net_ratelimit())
		printk(KERN_WARNING "dst cache overflow\n");
	RT_CACHE_STAT_INC(gc_dst_overflow);
	return 1;

work_done:
	expire += ip_rt_gc_min_interval;
	if (expire > ip_rt_gc_timeout ||
1069 1070
	    dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
	    dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
L
Linus Torvalds 已提交
1071 1072 1073
		expire = ip_rt_gc_timeout;
#if RT_CACHE_DEBUG >= 2
	printk(KERN_DEBUG "expire++ %u %d %d %d\n", expire,
1074
			dst_entries_get_fast(&ipv4_dst_ops), goal, rover);
L
Linus Torvalds 已提交
1075 1076 1077 1078
#endif
out:	return 0;
}

E
Eric Dumazet 已提交
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
/*
 * Returns number of entries in a hash chain that have different hash_inputs
 */
static int slow_chain_length(const struct rtable *head)
{
	int length = 0;
	const struct rtable *rth = head;

	while (rth) {
		length += has_noalias(head, rth);
1089
		rth = rcu_dereference_protected(rth->dst.rt_next, 1);
E
Eric Dumazet 已提交
1090 1091 1092 1093
	}
	return length >> FRACT_BITS;
}

E
Eric Dumazet 已提交
1094
static int rt_intern_hash(unsigned hash, struct rtable *rt,
1095
			  struct rtable **rp, struct sk_buff *skb, int ifindex)
L
Linus Torvalds 已提交
1096
{
1097 1098
	struct rtable	*rth, *cand;
	struct rtable __rcu **rthp, **candp;
L
Linus Torvalds 已提交
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
	unsigned long	now;
	u32 		min_score;
	int		chain_length;
	int attempts = !in_softirq();

restart:
	chain_length = 0;
	min_score = ~(u32)0;
	cand = NULL;
	candp = NULL;
	now = jiffies;

1111
	if (!rt_caching(dev_net(rt->dst.dev))) {
1112 1113 1114 1115 1116 1117 1118 1119
		/*
		 * If we're not caching, just tell the caller we
		 * were successful and don't touch the route.  The
		 * caller hold the sole reference to the cache entry, and
		 * it will be released when the caller is done with it.
		 * If we drop it here, the callers have no way to resolve routes
		 * when we're not caching.  Instead, just point *rp at rt, so
		 * the caller gets a single use out of the route
1120 1121 1122
		 * Note that we do rt_free on this new route entry, so that
		 * once its refcount hits zero, we are still able to reap it
		 * (Thanks Alexey)
E
Eric Dumazet 已提交
1123 1124 1125
		 * Note: To avoid expensive rcu stuff for this uncached dst,
		 * we set DST_NOCACHE so that dst_release() can free dst without
		 * waiting a grace period.
1126
		 */
1127

E
Eric Dumazet 已提交
1128
		rt->dst.flags |= DST_NOCACHE;
1129
		if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1130
			int err = arp_bind_neighbour(&rt->dst);
1131 1132 1133 1134
			if (err) {
				if (net_ratelimit())
					printk(KERN_WARNING
					    "Neighbour table failure & not caching routes.\n");
E
Eric Dumazet 已提交
1135
				ip_rt_put(rt);
1136 1137 1138 1139 1140
				return err;
			}
		}

		goto skip_hashing;
1141 1142
	}

L
Linus Torvalds 已提交
1143 1144
	rthp = &rt_hash_table[hash].chain;

1145
	spin_lock_bh(rt_hash_lock_addr(hash));
1146 1147
	while ((rth = rcu_dereference_protected(*rthp,
			lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1148
		if (rt_is_expired(rth)) {
1149
			*rthp = rth->dst.rt_next;
1150 1151 1152
			rt_free(rth);
			continue;
		}
1153
		if (compare_keys(&rth->fl, &rt->fl) && compare_netns(rth, rt)) {
L
Linus Torvalds 已提交
1154
			/* Put it first */
1155
			*rthp = rth->dst.rt_next;
L
Linus Torvalds 已提交
1156 1157 1158 1159 1160
			/*
			 * Since lookup is lockfree, the deletion
			 * must be visible to another weakly ordered CPU before
			 * the insertion at the start of the hash chain.
			 */
1161
			rcu_assign_pointer(rth->dst.rt_next,
L
Linus Torvalds 已提交
1162 1163 1164 1165 1166 1167 1168
					   rt_hash_table[hash].chain);
			/*
			 * Since lookup is lockfree, the update writes
			 * must be ordered for consistency on SMP.
			 */
			rcu_assign_pointer(rt_hash_table[hash].chain, rth);

1169
			dst_use(&rth->dst, now);
1170
			spin_unlock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1171 1172

			rt_drop(rt);
E
Eric Dumazet 已提交
1173 1174 1175
			if (rp)
				*rp = rth;
			else
1176
				skb_dst_set(skb, &rth->dst);
L
Linus Torvalds 已提交
1177 1178 1179
			return 0;
		}

1180
		if (!atomic_read(&rth->dst.__refcnt)) {
L
Linus Torvalds 已提交
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
			u32 score = rt_score(rth);

			if (score <= min_score) {
				cand = rth;
				candp = rthp;
				min_score = score;
			}
		}

		chain_length++;

1192
		rthp = &rth->dst.rt_next;
L
Linus Torvalds 已提交
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202
	}

	if (cand) {
		/* ip_rt_gc_elasticity used to be average length of chain
		 * length, when exceeded gc becomes really aggressive.
		 *
		 * The second limit is less certain. At the moment it allows
		 * only 2 entries per bucket. We will see.
		 */
		if (chain_length > ip_rt_gc_elasticity) {
1203
			*candp = cand->dst.rt_next;
L
Linus Torvalds 已提交
1204 1205
			rt_free(cand);
		}
1206
	} else {
E
Eric Dumazet 已提交
1207 1208
		if (chain_length > rt_chain_length_max &&
		    slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1209
			struct net *net = dev_net(rt->dst.dev);
1210
			int num = ++net->ipv4.current_rt_cache_rebuild_count;
1211
			if (!rt_caching(net)) {
1212
				printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
1213
					rt->dst.dev->name, num);
1214
			}
1215
			rt_emergency_hash_rebuild(net);
1216 1217 1218 1219 1220
			spin_unlock_bh(rt_hash_lock_addr(hash));

			hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
					ifindex, rt_genid(net));
			goto restart;
1221
		}
L
Linus Torvalds 已提交
1222 1223 1224 1225 1226
	}

	/* Try to bind route to arp only if it is output
	   route or unicast forwarding path.
	 */
1227
	if (rt->rt_type == RTN_UNICAST || rt_is_output_route(rt)) {
1228
		int err = arp_bind_neighbour(&rt->dst);
L
Linus Torvalds 已提交
1229
		if (err) {
1230
			spin_unlock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245

			if (err != -ENOBUFS) {
				rt_drop(rt);
				return err;
			}

			/* Neighbour tables are full and nothing
			   can be released. Try to shrink route cache,
			   it is most likely it holds some neighbour records.
			 */
			if (attempts-- > 0) {
				int saved_elasticity = ip_rt_gc_elasticity;
				int saved_int = ip_rt_gc_min_interval;
				ip_rt_gc_elasticity	= 1;
				ip_rt_gc_min_interval	= 0;
1246
				rt_garbage_collect(&ipv4_dst_ops);
L
Linus Torvalds 已提交
1247 1248 1249 1250 1251 1252
				ip_rt_gc_min_interval	= saved_int;
				ip_rt_gc_elasticity	= saved_elasticity;
				goto restart;
			}

			if (net_ratelimit())
1253
				printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
L
Linus Torvalds 已提交
1254 1255 1256 1257 1258
			rt_drop(rt);
			return -ENOBUFS;
		}
	}

1259
	rt->dst.rt_next = rt_hash_table[hash].chain;
1260

L
Linus Torvalds 已提交
1261
#if RT_CACHE_DEBUG >= 2
1262
	if (rt->dst.rt_next) {
L
Linus Torvalds 已提交
1263
		struct rtable *trt;
1264 1265
		printk(KERN_DEBUG "rt_cache @%02x: %pI4",
		       hash, &rt->rt_dst);
1266
		for (trt = rt->dst.rt_next; trt; trt = trt->dst.rt_next)
1267
			printk(" . %pI4", &trt->rt_dst);
L
Linus Torvalds 已提交
1268 1269 1270
		printk("\n");
	}
#endif
1271 1272 1273 1274 1275
	/*
	 * Since lookup is lockfree, we must make sure
	 * previous writes to rt are comitted to memory
	 * before making rt visible to other CPUS.
	 */
1276
	rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1277

1278
	spin_unlock_bh(rt_hash_lock_addr(hash));
1279

1280
skip_hashing:
E
Eric Dumazet 已提交
1281 1282 1283
	if (rp)
		*rp = rt;
	else
1284
		skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
1285 1286 1287 1288 1289 1290 1291
	return 0;
}

void rt_bind_peer(struct rtable *rt, int create)
{
	struct inet_peer *peer;

1292
	peer = inet_getpeer_v4(rt->rt_dst, create);
L
Linus Torvalds 已提交
1293

1294
	if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
L
Linus Torvalds 已提交
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
		inet_putpeer(peer);
}

/*
 * Peer allocation may fail only in serious out-of-memory conditions.  However
 * we still can generate some output.
 * Random ID selection looks a bit dangerous because we have no chances to
 * select ID being unique in a reasonable period of time.
 * But broken packet identifier may be better than no packet at all.
 */
static void ip_select_fb_ident(struct iphdr *iph)
{
	static DEFINE_SPINLOCK(ip_fb_id_lock);
	static u32 ip_fallback_id;
	u32 salt;

	spin_lock_bh(&ip_fb_id_lock);
1312
	salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
L
Linus Torvalds 已提交
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
	iph->id = htons(salt & 0xFFFF);
	ip_fallback_id = salt;
	spin_unlock_bh(&ip_fb_id_lock);
}

void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
{
	struct rtable *rt = (struct rtable *) dst;

	if (rt) {
		if (rt->peer == NULL)
			rt_bind_peer(rt, 1);

		/* If peer is attached to destination, it is never detached,
		   so that we need not to grab a lock to dereference it.
		 */
		if (rt->peer) {
			iph->id = htons(inet_getid(rt->peer, more));
			return;
		}
	} else
1334
		printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
1335
		       __builtin_return_address(0));
L
Linus Torvalds 已提交
1336 1337 1338

	ip_select_fb_ident(iph);
}
E
Eric Dumazet 已提交
1339
EXPORT_SYMBOL(__ip_select_ident);
L
Linus Torvalds 已提交
1340 1341 1342

static void rt_del(unsigned hash, struct rtable *rt)
{
1343 1344
	struct rtable __rcu **rthp;
	struct rtable *aux;
L
Linus Torvalds 已提交
1345

1346
	rthp = &rt_hash_table[hash].chain;
1347
	spin_lock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1348
	ip_rt_put(rt);
1349 1350
	while ((aux = rcu_dereference_protected(*rthp,
			lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1351
		if (aux == rt || rt_is_expired(aux)) {
1352
			*rthp = aux->dst.rt_next;
1353 1354
			rt_free(aux);
			continue;
L
Linus Torvalds 已提交
1355
		}
1356
		rthp = &aux->dst.rt_next;
1357
	}
1358
	spin_unlock_bh(rt_hash_lock_addr(hash));
L
Linus Torvalds 已提交
1359 1360
}

1361
/* called in rcu_read_lock() section */
A
Al Viro 已提交
1362 1363
void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
		    __be32 saddr, struct net_device *dev)
L
Linus Torvalds 已提交
1364 1365
{
	int i, k;
1366
	struct in_device *in_dev = __in_dev_get_rcu(dev);
1367 1368
	struct rtable *rth;
	struct rtable __rcu **rthp;
A
Al Viro 已提交
1369
	__be32  skeys[2] = { saddr, 0 };
L
Linus Torvalds 已提交
1370
	int  ikeys[2] = { dev->ifindex, 0 };
1371
	struct netevent_redirect netevent;
1372
	struct net *net;
L
Linus Torvalds 已提交
1373 1374 1375 1376

	if (!in_dev)
		return;

1377
	net = dev_net(dev);
1378 1379 1380
	if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
	    ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
	    ipv4_is_zeronet(new_gw))
L
Linus Torvalds 已提交
1381 1382
		goto reject_redirect;

1383 1384 1385
	if (!rt_caching(net))
		goto reject_redirect;

L
Linus Torvalds 已提交
1386 1387 1388 1389 1390 1391
	if (!IN_DEV_SHARED_MEDIA(in_dev)) {
		if (!inet_addr_onlink(in_dev, new_gw, old_gw))
			goto reject_redirect;
		if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
			goto reject_redirect;
	} else {
1392
		if (inet_addr_type(net, new_gw) != RTN_UNICAST)
L
Linus Torvalds 已提交
1393 1394 1395 1396 1397
			goto reject_redirect;
	}

	for (i = 0; i < 2; i++) {
		for (k = 0; k < 2; k++) {
1398
			unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1399
						rt_genid(net));
L
Linus Torvalds 已提交
1400

1401
			rthp = &rt_hash_table[hash].chain;
L
Linus Torvalds 已提交
1402 1403 1404 1405 1406 1407 1408

			while ((rth = rcu_dereference(*rthp)) != NULL) {
				struct rtable *rt;

				if (rth->fl.fl4_dst != daddr ||
				    rth->fl.fl4_src != skeys[i] ||
				    rth->fl.oif != ikeys[k] ||
1409
				    rt_is_input_route(rth) ||
1410
				    rt_is_expired(rth) ||
1411 1412
				    !net_eq(dev_net(rth->dst.dev), net)) {
					rthp = &rth->dst.rt_next;
L
Linus Torvalds 已提交
1413 1414 1415 1416 1417
					continue;
				}

				if (rth->rt_dst != daddr ||
				    rth->rt_src != saddr ||
1418
				    rth->dst.error ||
L
Linus Torvalds 已提交
1419
				    rth->rt_gateway != old_gw ||
1420
				    rth->dst.dev != dev)
L
Linus Torvalds 已提交
1421 1422
					break;

1423
				dst_hold(&rth->dst);
L
Linus Torvalds 已提交
1424 1425 1426 1427 1428 1429 1430 1431 1432

				rt = dst_alloc(&ipv4_dst_ops);
				if (rt == NULL) {
					ip_rt_put(rth);
					return;
				}

				/* Copy all the information. */
				*rt = *rth;
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
				rt->dst.__use		= 1;
				atomic_set(&rt->dst.__refcnt, 1);
				rt->dst.child		= NULL;
				if (rt->dst.dev)
					dev_hold(rt->dst.dev);
				rt->dst.obsolete	= -1;
				rt->dst.lastuse	= jiffies;
				rt->dst.path		= &rt->dst;
				rt->dst.neighbour	= NULL;
				rt->dst.hh		= NULL;
1443
#ifdef CONFIG_XFRM
1444
				rt->dst.xfrm		= NULL;
1445
#endif
1446
				rt->rt_genid		= rt_genid(net);
L
Linus Torvalds 已提交
1447 1448 1449 1450 1451 1452
				rt->rt_flags		|= RTCF_REDIRECTED;

				/* Gateway is different ... */
				rt->rt_gateway		= new_gw;

				/* Redirect received -> path was valid */
1453
				dst_confirm(&rth->dst);
L
Linus Torvalds 已提交
1454 1455 1456 1457

				if (rt->peer)
					atomic_inc(&rt->peer->refcnt);

1458 1459
				if (arp_bind_neighbour(&rt->dst) ||
				    !(rt->dst.neighbour->nud_state &
L
Linus Torvalds 已提交
1460
					    NUD_VALID)) {
1461 1462
					if (rt->dst.neighbour)
						neigh_event_send(rt->dst.neighbour, NULL);
L
Linus Torvalds 已提交
1463 1464 1465 1466
					ip_rt_put(rth);
					rt_drop(rt);
					goto do_next;
				}
1467

1468 1469
				netevent.old = &rth->dst;
				netevent.new = &rt->dst;
1470 1471
				call_netevent_notifiers(NETEVENT_REDIRECT,
							&netevent);
L
Linus Torvalds 已提交
1472 1473

				rt_del(hash, rth);
1474
				if (!rt_intern_hash(hash, rt, &rt, NULL, rt->fl.oif))
L
Linus Torvalds 已提交
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
					ip_rt_put(rt);
				goto do_next;
			}
		do_next:
			;
		}
	}
	return;

reject_redirect:
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
1487 1488 1489 1490
		printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
			"  Advised path = %pI4 -> %pI4\n",
		       &old_gw, dev->name, &new_gw,
		       &saddr, &daddr);
L
Linus Torvalds 已提交
1491
#endif
1492
	;
L
Linus Torvalds 已提交
1493 1494 1495 1496
}

static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
{
1497
	struct rtable *rt = (struct rtable *)dst;
L
Linus Torvalds 已提交
1498 1499 1500
	struct dst_entry *ret = dst;

	if (rt) {
T
Timo Teräs 已提交
1501
		if (dst->obsolete > 0) {
L
Linus Torvalds 已提交
1502 1503 1504
			ip_rt_put(rt);
			ret = NULL;
		} else if ((rt->rt_flags & RTCF_REDIRECTED) ||
1505 1506
			   (rt->dst.expires &&
			    time_after_eq(jiffies, rt->dst.expires))) {
A
Al Viro 已提交
1507
			unsigned hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1508
						rt->fl.oif,
1509
						rt_genid(dev_net(dst->dev)));
L
Linus Torvalds 已提交
1510
#if RT_CACHE_DEBUG >= 1
1511 1512
			printk(KERN_DEBUG "ipv4_negative_advice: redirect to %pI4/%02x dropped\n",
				&rt->rt_dst, rt->fl.fl4_tos);
L
Linus Torvalds 已提交
1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
#endif
			rt_del(hash, rt);
			ret = NULL;
		}
	}
	return ret;
}

/*
 * Algorithm:
 *	1. The first ip_rt_redirect_number redirects are sent
 *	   with exponential backoff, then we stop sending them at all,
 *	   assuming that the host ignores our redirects.
 *	2. If we did not see packets requiring redirects
 *	   during ip_rt_redirect_silence, we assume that the host
 *	   forgot redirected route and start to send redirects again.
 *
 * This algorithm is much cheaper and more intelligent than dumb load limiting
 * in icmp.c.
 *
 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 * and "frag. need" (breaks PMTU discovery) in icmp.c.
 */

void ip_rt_send_redirect(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
1539
	struct rtable *rt = skb_rtable(skb);
1540 1541
	struct in_device *in_dev;
	int log_martians;
L
Linus Torvalds 已提交
1542

1543
	rcu_read_lock();
1544
	in_dev = __in_dev_get_rcu(rt->dst.dev);
1545 1546
	if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
		rcu_read_unlock();
L
Linus Torvalds 已提交
1547
		return;
1548 1549 1550
	}
	log_martians = IN_DEV_LOG_MARTIANS(in_dev);
	rcu_read_unlock();
L
Linus Torvalds 已提交
1551 1552 1553 1554

	/* No redirected packets during ip_rt_redirect_silence;
	 * reset the algorithm.
	 */
1555 1556
	if (time_after(jiffies, rt->dst.rate_last + ip_rt_redirect_silence))
		rt->dst.rate_tokens = 0;
L
Linus Torvalds 已提交
1557 1558

	/* Too many ignored redirects; do not send anything
1559
	 * set dst.rate_last to the last seen redirected packet.
L
Linus Torvalds 已提交
1560
	 */
1561 1562
	if (rt->dst.rate_tokens >= ip_rt_redirect_number) {
		rt->dst.rate_last = jiffies;
1563
		return;
L
Linus Torvalds 已提交
1564 1565 1566 1567 1568
	}

	/* Check for load limit; set rate_last to the latest sent
	 * redirect.
	 */
1569
	if (rt->dst.rate_tokens == 0 ||
1570
	    time_after(jiffies,
1571 1572
		       (rt->dst.rate_last +
			(ip_rt_redirect_load << rt->dst.rate_tokens)))) {
L
Linus Torvalds 已提交
1573
		icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1574 1575
		rt->dst.rate_last = jiffies;
		++rt->dst.rate_tokens;
L
Linus Torvalds 已提交
1576
#ifdef CONFIG_IP_ROUTE_VERBOSE
1577
		if (log_martians &&
1578
		    rt->dst.rate_tokens == ip_rt_redirect_number &&
L
Linus Torvalds 已提交
1579
		    net_ratelimit())
1580 1581 1582
			printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
				&rt->rt_src, rt->rt_iif,
				&rt->rt_dst, &rt->rt_gateway);
L
Linus Torvalds 已提交
1583 1584 1585 1586 1587 1588
#endif
	}
}

static int ip_error(struct sk_buff *skb)
{
E
Eric Dumazet 已提交
1589
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
1590 1591 1592
	unsigned long now;
	int code;

1593
	switch (rt->dst.error) {
L
Linus Torvalds 已提交
1594 1595 1596 1597 1598 1599 1600 1601
		case EINVAL:
		default:
			goto out;
		case EHOSTUNREACH:
			code = ICMP_HOST_UNREACH;
			break;
		case ENETUNREACH:
			code = ICMP_NET_UNREACH;
1602
			IP_INC_STATS_BH(dev_net(rt->dst.dev),
1603
					IPSTATS_MIB_INNOROUTES);
L
Linus Torvalds 已提交
1604 1605 1606 1607 1608 1609 1610
			break;
		case EACCES:
			code = ICMP_PKT_FILTERED;
			break;
	}

	now = jiffies;
1611 1612 1613 1614 1615 1616
	rt->dst.rate_tokens += now - rt->dst.rate_last;
	if (rt->dst.rate_tokens > ip_rt_error_burst)
		rt->dst.rate_tokens = ip_rt_error_burst;
	rt->dst.rate_last = now;
	if (rt->dst.rate_tokens >= ip_rt_error_cost) {
		rt->dst.rate_tokens -= ip_rt_error_cost;
L
Linus Torvalds 已提交
1617 1618 1619 1620 1621
		icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
	}

out:	kfree_skb(skb);
	return 0;
1622
}
L
Linus Torvalds 已提交
1623 1624 1625 1626 1627 1628

/*
 *	The last two values are not from the RFC but
 *	are needed for AMPRnet AX.25 paths.
 */

1629
static const unsigned short mtu_plateau[] =
L
Linus Torvalds 已提交
1630 1631
{32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };

S
Stephen Hemminger 已提交
1632
static inline unsigned short guess_mtu(unsigned short old_mtu)
L
Linus Torvalds 已提交
1633 1634
{
	int i;
1635

L
Linus Torvalds 已提交
1636 1637 1638 1639 1640 1641
	for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
		if (old_mtu > mtu_plateau[i])
			return mtu_plateau[i];
	return 68;
}

1642
unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
1643 1644
				 unsigned short new_mtu,
				 struct net_device *dev)
L
Linus Torvalds 已提交
1645
{
1646
	int i, k;
L
Linus Torvalds 已提交
1647 1648
	unsigned short old_mtu = ntohs(iph->tot_len);
	struct rtable *rth;
1649
	int  ikeys[2] = { dev->ifindex, 0 };
1650 1651
	__be32  skeys[2] = { iph->saddr, 0, };
	__be32  daddr = iph->daddr;
L
Linus Torvalds 已提交
1652 1653
	unsigned short est_mtu = 0;

1654 1655
	for (k = 0; k < 2; k++) {
		for (i = 0; i < 2; i++) {
1656
			unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1657
						rt_genid(net));
1658 1659 1660

			rcu_read_lock();
			for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
1661
			     rth = rcu_dereference(rth->dst.rt_next)) {
L
Linus Torvalds 已提交
1662 1663
				unsigned short mtu = new_mtu;

1664 1665 1666 1667 1668
				if (rth->fl.fl4_dst != daddr ||
				    rth->fl.fl4_src != skeys[i] ||
				    rth->rt_dst != daddr ||
				    rth->rt_src != iph->saddr ||
				    rth->fl.oif != ikeys[k] ||
1669
				    rt_is_input_route(rth) ||
1670 1671
				    dst_metric_locked(&rth->dst, RTAX_MTU) ||
				    !net_eq(dev_net(rth->dst.dev), net) ||
1672
				    rt_is_expired(rth))
1673 1674
					continue;

L
Linus Torvalds 已提交
1675 1676 1677 1678
				if (new_mtu < 68 || new_mtu >= old_mtu) {

					/* BSD 4.2 compatibility hack :-( */
					if (mtu == 0 &&
1679
					    old_mtu >= dst_mtu(&rth->dst) &&
L
Linus Torvalds 已提交
1680 1681 1682 1683 1684
					    old_mtu >= 68 + (iph->ihl << 2))
						old_mtu -= iph->ihl << 2;

					mtu = guess_mtu(old_mtu);
				}
1685 1686 1687
				if (mtu <= dst_mtu(&rth->dst)) {
					if (mtu < dst_mtu(&rth->dst)) {
						dst_confirm(&rth->dst);
L
Linus Torvalds 已提交
1688
						if (mtu < ip_rt_min_pmtu) {
1689 1690
							u32 lock = dst_metric(&rth->dst,
									      RTAX_LOCK);
L
Linus Torvalds 已提交
1691
							mtu = ip_rt_min_pmtu;
1692 1693 1694
							lock |= (1 << RTAX_MTU);
							dst_metric_set(&rth->dst, RTAX_LOCK,
								       lock);
L
Linus Torvalds 已提交
1695
						}
1696
						dst_metric_set(&rth->dst, RTAX_MTU, mtu);
1697
						dst_set_expires(&rth->dst,
L
Linus Torvalds 已提交
1698 1699 1700 1701 1702
							ip_rt_mtu_expires);
					}
					est_mtu = mtu;
				}
			}
1703
			rcu_read_unlock();
L
Linus Torvalds 已提交
1704 1705 1706 1707 1708 1709 1710
		}
	}
	return est_mtu ? : new_mtu;
}

static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
{
1711
	if (dst_mtu(dst) > mtu && mtu >= 68 &&
L
Linus Torvalds 已提交
1712 1713
	    !(dst_metric_locked(dst, RTAX_MTU))) {
		if (mtu < ip_rt_min_pmtu) {
1714
			u32 lock = dst_metric(dst, RTAX_LOCK);
L
Linus Torvalds 已提交
1715
			mtu = ip_rt_min_pmtu;
1716
			dst_metric_set(dst, RTAX_LOCK, lock | (1 << RTAX_MTU));
L
Linus Torvalds 已提交
1717
		}
1718
		dst_metric_set(dst, RTAX_MTU, mtu);
L
Linus Torvalds 已提交
1719
		dst_set_expires(dst, ip_rt_mtu_expires);
1720
		call_netevent_notifiers(NETEVENT_PMTU_UPDATE, dst);
L
Linus Torvalds 已提交
1721 1722 1723 1724 1725
	}
}

static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
{
T
Timo Teräs 已提交
1726 1727 1728
	if (rt_is_expired((struct rtable *)dst))
		return NULL;
	return dst;
L
Linus Torvalds 已提交
1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748
}

static void ipv4_dst_destroy(struct dst_entry *dst)
{
	struct rtable *rt = (struct rtable *) dst;
	struct inet_peer *peer = rt->peer;

	if (peer) {
		rt->peer = NULL;
		inet_putpeer(peer);
	}
}


static void ipv4_link_failure(struct sk_buff *skb)
{
	struct rtable *rt;

	icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);

E
Eric Dumazet 已提交
1749
	rt = skb_rtable(skb);
L
Linus Torvalds 已提交
1750
	if (rt)
1751
		dst_set_expires(&rt->dst, 0);
L
Linus Torvalds 已提交
1752 1753 1754 1755
}

static int ip_rt_bug(struct sk_buff *skb)
{
1756 1757
	printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
		&ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
L
Linus Torvalds 已提交
1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
		skb->dev ? skb->dev->name : "?");
	kfree_skb(skb);
	return 0;
}

/*
   We do not cache source address of outgoing interface,
   because it is used only by IP RR, TS and SRR options,
   so that it out of fast path.

   BTW remember: "addr" is allowed to be not aligned
   in IP options!
 */

void ip_rt_get_source(u8 *addr, struct rtable *rt)
{
A
Al Viro 已提交
1774
	__be32 src;
L
Linus Torvalds 已提交
1775 1776
	struct fib_result res;

1777
	if (rt_is_output_route(rt))
L
Linus Torvalds 已提交
1778
		src = rt->rt_src;
E
Eric Dumazet 已提交
1779 1780 1781 1782 1783 1784
	else {
		rcu_read_lock();
		if (fib_lookup(dev_net(rt->dst.dev), &rt->fl, &res) == 0)
			src = FIB_RES_PREFSRC(res);
		else
			src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
L
Linus Torvalds 已提交
1785
					RT_SCOPE_UNIVERSE);
E
Eric Dumazet 已提交
1786 1787
		rcu_read_unlock();
	}
L
Linus Torvalds 已提交
1788 1789 1790 1791 1792 1793
	memcpy(addr, &src, 4);
}

#ifdef CONFIG_NET_CLS_ROUTE
static void set_class_tag(struct rtable *rt, u32 tag)
{
1794 1795 1796 1797
	if (!(rt->dst.tclassid & 0xFFFF))
		rt->dst.tclassid |= tag & 0xFFFF;
	if (!(rt->dst.tclassid & 0xFFFF0000))
		rt->dst.tclassid |= tag & 0xFFFF0000;
L
Linus Torvalds 已提交
1798 1799 1800 1801 1802
}
#endif

static void rt_set_nexthop(struct rtable *rt, struct fib_result *res, u32 itag)
{
1803
	struct dst_entry *dst = &rt->dst;
L
Linus Torvalds 已提交
1804 1805 1806 1807 1808 1809
	struct fib_info *fi = res->fi;

	if (fi) {
		if (FIB_RES_GW(*res) &&
		    FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
			rt->rt_gateway = FIB_RES_GW(*res);
1810
		dst_import_metrics(dst, fi->fib_metrics);
L
Linus Torvalds 已提交
1811
		if (fi->fib_mtu == 0) {
1812 1813
			dst_metric_set(dst, RTAX_MTU, dst->dev->mtu);
			if (dst_metric_locked(dst, RTAX_MTU) &&
L
Linus Torvalds 已提交
1814
			    rt->rt_gateway != rt->rt_dst &&
1815 1816
			    dst->dev->mtu > 576)
				dst_metric_set(dst, RTAX_MTU, 576);
L
Linus Torvalds 已提交
1817 1818
		}
#ifdef CONFIG_NET_CLS_ROUTE
1819
		dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
L
Linus Torvalds 已提交
1820 1821
#endif
	} else
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831
		dst_metric_set(dst, RTAX_MTU, dst->dev->mtu);

	if (dst_mtu(dst) > IP_MAX_MTU)
		dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
	if (dst_metric(dst, RTAX_ADVMSS) == 0)
		dst_metric_set(dst, RTAX_ADVMSS,
			       max_t(unsigned int, dst->dev->mtu - 40,
				     ip_rt_min_advmss));
	if (dst_metric(dst, RTAX_ADVMSS) > 65535 - 40)
		dst_metric_set(dst, RTAX_ADVMSS, 65535 - 40);
L
Linus Torvalds 已提交
1832 1833 1834 1835 1836 1837 1838

#ifdef CONFIG_NET_CLS_ROUTE
#ifdef CONFIG_IP_MULTIPLE_TABLES
	set_class_tag(rt, fib_rules_tclass(res));
#endif
	set_class_tag(rt, itag);
#endif
1839
	rt->rt_type = res->type;
L
Linus Torvalds 已提交
1840 1841
}

1842
/* called in rcu_read_lock() section */
A
Al Viro 已提交
1843
static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
L
Linus Torvalds 已提交
1844 1845
				u8 tos, struct net_device *dev, int our)
{
1846
	unsigned int hash;
L
Linus Torvalds 已提交
1847
	struct rtable *rth;
A
Al Viro 已提交
1848
	__be32 spec_dst;
1849
	struct in_device *in_dev = __in_dev_get_rcu(dev);
L
Linus Torvalds 已提交
1850
	u32 itag = 0;
1851
	int err;
L
Linus Torvalds 已提交
1852 1853 1854 1855 1856 1857

	/* Primary sanity checks. */

	if (in_dev == NULL)
		return -EINVAL;

1858
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1859
	    ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
L
Linus Torvalds 已提交
1860 1861
		goto e_inval;

1862 1863
	if (ipv4_is_zeronet(saddr)) {
		if (!ipv4_is_local_multicast(daddr))
L
Linus Torvalds 已提交
1864 1865
			goto e_inval;
		spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
1866 1867 1868 1869 1870 1871
	} else {
		err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
					  &itag, 0);
		if (err < 0)
			goto e_err;
	}
L
Linus Torvalds 已提交
1872 1873 1874 1875
	rth = dst_alloc(&ipv4_dst_ops);
	if (!rth)
		goto e_nobufs;

1876 1877
	rth->dst.output = ip_rt_bug;
	rth->dst.obsolete = -1;
L
Linus Torvalds 已提交
1878

1879 1880
	atomic_set(&rth->dst.__refcnt, 1);
	rth->dst.flags= DST_HOST;
1881
	if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
1882
		rth->dst.flags |= DST_NOPOLICY;
L
Linus Torvalds 已提交
1883 1884 1885
	rth->fl.fl4_dst	= daddr;
	rth->rt_dst	= daddr;
	rth->fl.fl4_tos	= tos;
1886
	rth->fl.mark    = skb->mark;
L
Linus Torvalds 已提交
1887 1888 1889
	rth->fl.fl4_src	= saddr;
	rth->rt_src	= saddr;
#ifdef CONFIG_NET_CLS_ROUTE
1890
	rth->dst.tclassid = itag;
L
Linus Torvalds 已提交
1891 1892 1893
#endif
	rth->rt_iif	=
	rth->fl.iif	= dev->ifindex;
1894 1895
	rth->dst.dev	= init_net.loopback_dev;
	dev_hold(rth->dst.dev);
L
Linus Torvalds 已提交
1896 1897 1898
	rth->fl.oif	= 0;
	rth->rt_gateway	= daddr;
	rth->rt_spec_dst= spec_dst;
1899
	rth->rt_genid	= rt_genid(dev_net(dev));
L
Linus Torvalds 已提交
1900
	rth->rt_flags	= RTCF_MULTICAST;
1901
	rth->rt_type	= RTN_MULTICAST;
L
Linus Torvalds 已提交
1902
	if (our) {
1903
		rth->dst.input= ip_local_deliver;
L
Linus Torvalds 已提交
1904 1905 1906 1907
		rth->rt_flags |= RTCF_LOCAL;
	}

#ifdef CONFIG_IP_MROUTE
1908
	if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1909
		rth->dst.input = ip_mr_input;
L
Linus Torvalds 已提交
1910 1911 1912
#endif
	RT_CACHE_STAT_INC(in_slow_mc);

1913
	hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1914
	return rt_intern_hash(hash, rth, NULL, skb, dev->ifindex);
L
Linus Torvalds 已提交
1915 1916 1917 1918

e_nobufs:
	return -ENOBUFS;
e_inval:
1919
	return -EINVAL;
1920 1921
e_err:
	return err;
L
Linus Torvalds 已提交
1922 1923 1924 1925 1926 1927
}


static void ip_handle_martian_source(struct net_device *dev,
				     struct in_device *in_dev,
				     struct sk_buff *skb,
A
Al Viro 已提交
1928 1929
				     __be32 daddr,
				     __be32 saddr)
L
Linus Torvalds 已提交
1930 1931 1932 1933 1934 1935 1936 1937
{
	RT_CACHE_STAT_INC(in_martian_src);
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
		/*
		 *	RFC1812 recommendation, if source is martian,
		 *	the only hint is MAC header.
		 */
1938 1939
		printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
			&daddr, &saddr, dev->name);
1940
		if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
L
Linus Torvalds 已提交
1941
			int i;
1942
			const unsigned char *p = skb_mac_header(skb);
L
Linus Torvalds 已提交
1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954
			printk(KERN_WARNING "ll header: ");
			for (i = 0; i < dev->hard_header_len; i++, p++) {
				printk("%02x", *p);
				if (i < (dev->hard_header_len - 1))
					printk(":");
			}
			printk("\n");
		}
	}
#endif
}

1955
/* called in rcu_read_lock() section */
S
Stephen Hemminger 已提交
1956 1957 1958 1959 1960
static int __mkroute_input(struct sk_buff *skb,
			   struct fib_result *res,
			   struct in_device *in_dev,
			   __be32 daddr, __be32 saddr, u32 tos,
			   struct rtable **result)
L
Linus Torvalds 已提交
1961 1962 1963 1964
{
	struct rtable *rth;
	int err;
	struct in_device *out_dev;
1965
	unsigned int flags = 0;
1966 1967
	__be32 spec_dst;
	u32 itag;
L
Linus Torvalds 已提交
1968 1969

	/* get a working reference to the output device */
1970
	out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
L
Linus Torvalds 已提交
1971 1972 1973 1974 1975 1976 1977 1978
	if (out_dev == NULL) {
		if (net_ratelimit())
			printk(KERN_CRIT "Bug in ip_route_input" \
			       "_slow(). Please, report\n");
		return -EINVAL;
	}


1979
	err = fib_validate_source(saddr, daddr, tos, FIB_RES_OIF(*res),
J
jamal 已提交
1980
				  in_dev->dev, &spec_dst, &itag, skb->mark);
L
Linus Torvalds 已提交
1981
	if (err < 0) {
1982
		ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
L
Linus Torvalds 已提交
1983
					 saddr);
1984

L
Linus Torvalds 已提交
1985 1986 1987 1988 1989 1990
		goto cleanup;
	}

	if (err)
		flags |= RTCF_DIRECTSRC;

1991
	if (out_dev == in_dev && err &&
L
Linus Torvalds 已提交
1992 1993 1994 1995 1996 1997 1998
	    (IN_DEV_SHARED_MEDIA(out_dev) ||
	     inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
		flags |= RTCF_DOREDIRECT;

	if (skb->protocol != htons(ETH_P_IP)) {
		/* Not IP (i.e. ARP). Do not create route, if it is
		 * invalid for proxy arp. DNAT routes are always valid.
1999 2000 2001 2002
		 *
		 * Proxy arp feature have been extended to allow, ARP
		 * replies back to the same interface, to support
		 * Private VLAN switch technologies. See arp.c.
L
Linus Torvalds 已提交
2003
		 */
2004 2005
		if (out_dev == in_dev &&
		    IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
L
Linus Torvalds 已提交
2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017
			err = -EINVAL;
			goto cleanup;
		}
	}


	rth = dst_alloc(&ipv4_dst_ops);
	if (!rth) {
		err = -ENOBUFS;
		goto cleanup;
	}

2018 2019
	atomic_set(&rth->dst.__refcnt, 1);
	rth->dst.flags= DST_HOST;
2020
	if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2021
		rth->dst.flags |= DST_NOPOLICY;
2022
	if (IN_DEV_CONF_GET(out_dev, NOXFRM))
2023
		rth->dst.flags |= DST_NOXFRM;
L
Linus Torvalds 已提交
2024 2025 2026
	rth->fl.fl4_dst	= daddr;
	rth->rt_dst	= daddr;
	rth->fl.fl4_tos	= tos;
2027
	rth->fl.mark    = skb->mark;
L
Linus Torvalds 已提交
2028 2029 2030 2031 2032
	rth->fl.fl4_src	= saddr;
	rth->rt_src	= saddr;
	rth->rt_gateway	= daddr;
	rth->rt_iif 	=
		rth->fl.iif	= in_dev->dev->ifindex;
2033 2034
	rth->dst.dev	= (out_dev)->dev;
	dev_hold(rth->dst.dev);
L
Linus Torvalds 已提交
2035 2036 2037
	rth->fl.oif 	= 0;
	rth->rt_spec_dst= spec_dst;

2038 2039 2040 2041
	rth->dst.obsolete = -1;
	rth->dst.input = ip_forward;
	rth->dst.output = ip_output;
	rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
L
Linus Torvalds 已提交
2042 2043 2044 2045 2046 2047 2048 2049 2050

	rt_set_nexthop(rth, res, itag);

	rth->rt_flags = flags;

	*result = rth;
	err = 0;
 cleanup:
	return err;
2051
}
L
Linus Torvalds 已提交
2052

S
Stephen Hemminger 已提交
2053 2054 2055 2056 2057
static int ip_mkroute_input(struct sk_buff *skb,
			    struct fib_result *res,
			    const struct flowi *fl,
			    struct in_device *in_dev,
			    __be32 daddr, __be32 saddr, u32 tos)
L
Linus Torvalds 已提交
2058
{
C
Chuck Short 已提交
2059
	struct rtable* rth = NULL;
L
Linus Torvalds 已提交
2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073
	int err;
	unsigned hash;

#ifdef CONFIG_IP_ROUTE_MULTIPATH
	if (res->fi && res->fi->fib_nhs > 1 && fl->oif == 0)
		fib_select_multipath(fl, res);
#endif

	/* create a routing cache entry */
	err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
	if (err)
		return err;

	/* put it into the cache */
2074
	hash = rt_hash(daddr, saddr, fl->iif,
2075
		       rt_genid(dev_net(rth->dst.dev)));
2076
	return rt_intern_hash(hash, rth, NULL, skb, fl->iif);
L
Linus Torvalds 已提交
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086
}

/*
 *	NOTE. We drop all the packets that has local source
 *	addresses, because every properly looped back packet
 *	must have correct destination already attached by output routine.
 *
 *	Such approach solves two big problems:
 *	1. Not simplex devices are handled properly.
 *	2. IP spoofing attempts are filtered with 100% of guarantee.
E
Eric Dumazet 已提交
2087
 *	called with rcu_read_lock()
L
Linus Torvalds 已提交
2088 2089
 */

A
Al Viro 已提交
2090
static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
L
Linus Torvalds 已提交
2091 2092 2093
			       u8 tos, struct net_device *dev)
{
	struct fib_result res;
2094
	struct in_device *in_dev = __in_dev_get_rcu(dev);
2095 2096 2097 2098
	struct flowi fl = { .fl4_dst	= daddr,
			    .fl4_src	= saddr,
			    .fl4_tos	= tos,
			    .fl4_scope	= RT_SCOPE_UNIVERSE,
2099
			    .mark = skb->mark,
L
Linus Torvalds 已提交
2100 2101 2102 2103 2104
			    .iif = dev->ifindex };
	unsigned	flags = 0;
	u32		itag = 0;
	struct rtable * rth;
	unsigned	hash;
A
Al Viro 已提交
2105
	__be32		spec_dst;
L
Linus Torvalds 已提交
2106
	int		err = -EINVAL;
2107
	struct net    * net = dev_net(dev);
L
Linus Torvalds 已提交
2108 2109 2110 2111 2112 2113 2114 2115 2116 2117

	/* IP on this device is disabled. */

	if (!in_dev)
		goto out;

	/* Check for the most weird martians, which can be not detected
	   by fib_lookup.
	 */

2118
	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
2119
	    ipv4_is_loopback(saddr))
L
Linus Torvalds 已提交
2120 2121
		goto martian_source;

2122
	if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
L
Linus Torvalds 已提交
2123 2124 2125 2126 2127
		goto brd_input;

	/* Accept zero addresses only to limited broadcast;
	 * I even do not know to fix it or not. Waiting for complains :-)
	 */
2128
	if (ipv4_is_zeronet(saddr))
L
Linus Torvalds 已提交
2129 2130
		goto martian_source;

2131
	if (ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr))
L
Linus Torvalds 已提交
2132 2133 2134 2135 2136
		goto martian_destination;

	/*
	 *	Now we are ready to route packet.
	 */
E
Eric Dumazet 已提交
2137 2138
	err = fib_lookup(net, &fl, &res);
	if (err != 0) {
L
Linus Torvalds 已提交
2139
		if (!IN_DEV_FORWARD(in_dev))
2140
			goto e_hostunreach;
L
Linus Torvalds 已提交
2141 2142 2143 2144 2145 2146 2147 2148 2149
		goto no_route;
	}

	RT_CACHE_STAT_INC(in_slow_tot);

	if (res.type == RTN_BROADCAST)
		goto brd_input;

	if (res.type == RTN_LOCAL) {
2150
		err = fib_validate_source(saddr, daddr, tos,
E
Eric Dumazet 已提交
2151 2152
					  net->loopback_dev->ifindex,
					  dev, &spec_dst, &itag, skb->mark);
2153 2154 2155
		if (err < 0)
			goto martian_source_keep_err;
		if (err)
L
Linus Torvalds 已提交
2156 2157 2158 2159 2160 2161
			flags |= RTCF_DIRECTSRC;
		spec_dst = daddr;
		goto local_input;
	}

	if (!IN_DEV_FORWARD(in_dev))
2162
		goto e_hostunreach;
L
Linus Torvalds 已提交
2163 2164 2165 2166 2167 2168 2169 2170 2171 2172
	if (res.type != RTN_UNICAST)
		goto martian_destination;

	err = ip_mkroute_input(skb, &res, &fl, in_dev, daddr, saddr, tos);
out:	return err;

brd_input:
	if (skb->protocol != htons(ETH_P_IP))
		goto e_inval;

2173
	if (ipv4_is_zeronet(saddr))
L
Linus Torvalds 已提交
2174 2175 2176
		spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
	else {
		err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
J
jamal 已提交
2177
					  &itag, skb->mark);
L
Linus Torvalds 已提交
2178
		if (err < 0)
2179
			goto martian_source_keep_err;
L
Linus Torvalds 已提交
2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
		if (err)
			flags |= RTCF_DIRECTSRC;
	}
	flags |= RTCF_BROADCAST;
	res.type = RTN_BROADCAST;
	RT_CACHE_STAT_INC(in_brd);

local_input:
	rth = dst_alloc(&ipv4_dst_ops);
	if (!rth)
		goto e_nobufs;

2192 2193
	rth->dst.output= ip_rt_bug;
	rth->dst.obsolete = -1;
2194
	rth->rt_genid = rt_genid(net);
L
Linus Torvalds 已提交
2195

2196 2197
	atomic_set(&rth->dst.__refcnt, 1);
	rth->dst.flags= DST_HOST;
2198
	if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2199
		rth->dst.flags |= DST_NOPOLICY;
L
Linus Torvalds 已提交
2200 2201 2202
	rth->fl.fl4_dst	= daddr;
	rth->rt_dst	= daddr;
	rth->fl.fl4_tos	= tos;
2203
	rth->fl.mark    = skb->mark;
L
Linus Torvalds 已提交
2204 2205 2206
	rth->fl.fl4_src	= saddr;
	rth->rt_src	= saddr;
#ifdef CONFIG_NET_CLS_ROUTE
2207
	rth->dst.tclassid = itag;
L
Linus Torvalds 已提交
2208 2209 2210
#endif
	rth->rt_iif	=
	rth->fl.iif	= dev->ifindex;
2211 2212
	rth->dst.dev	= net->loopback_dev;
	dev_hold(rth->dst.dev);
L
Linus Torvalds 已提交
2213 2214
	rth->rt_gateway	= daddr;
	rth->rt_spec_dst= spec_dst;
2215
	rth->dst.input= ip_local_deliver;
L
Linus Torvalds 已提交
2216 2217
	rth->rt_flags 	= flags|RTCF_LOCAL;
	if (res.type == RTN_UNREACHABLE) {
2218 2219
		rth->dst.input= ip_error;
		rth->dst.error= -err;
L
Linus Torvalds 已提交
2220 2221 2222
		rth->rt_flags 	&= ~RTCF_LOCAL;
	}
	rth->rt_type	= res.type;
2223
	hash = rt_hash(daddr, saddr, fl.iif, rt_genid(net));
2224
	err = rt_intern_hash(hash, rth, NULL, skb, fl.iif);
E
Eric Dumazet 已提交
2225
	goto out;
L
Linus Torvalds 已提交
2226 2227 2228 2229 2230

no_route:
	RT_CACHE_STAT_INC(in_no_route);
	spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
	res.type = RTN_UNREACHABLE;
2231 2232
	if (err == -ESRCH)
		err = -ENETUNREACH;
L
Linus Torvalds 已提交
2233 2234 2235 2236 2237 2238 2239 2240 2241
	goto local_input;

	/*
	 *	Do not cache martian addresses: they should be logged (RFC1812)
	 */
martian_destination:
	RT_CACHE_STAT_INC(in_martian_dst);
#ifdef CONFIG_IP_ROUTE_VERBOSE
	if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
2242 2243
		printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
			&daddr, &saddr, dev->name);
L
Linus Torvalds 已提交
2244
#endif
2245 2246

e_hostunreach:
2247
	err = -EHOSTUNREACH;
E
Eric Dumazet 已提交
2248
	goto out;
2249

L
Linus Torvalds 已提交
2250 2251
e_inval:
	err = -EINVAL;
E
Eric Dumazet 已提交
2252
	goto out;
L
Linus Torvalds 已提交
2253 2254 2255

e_nobufs:
	err = -ENOBUFS;
E
Eric Dumazet 已提交
2256
	goto out;
L
Linus Torvalds 已提交
2257 2258

martian_source:
2259 2260
	err = -EINVAL;
martian_source_keep_err:
L
Linus Torvalds 已提交
2261
	ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
E
Eric Dumazet 已提交
2262
	goto out;
L
Linus Torvalds 已提交
2263 2264
}

2265 2266
int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
			   u8 tos, struct net_device *dev, bool noref)
L
Linus Torvalds 已提交
2267 2268 2269 2270
{
	struct rtable * rth;
	unsigned	hash;
	int iif = dev->ifindex;
2271
	struct net *net;
2272
	int res;
L
Linus Torvalds 已提交
2273

2274
	net = dev_net(dev);
2275

2276 2277
	rcu_read_lock();

2278 2279 2280
	if (!rt_caching(net))
		goto skip_cache;

L
Linus Torvalds 已提交
2281
	tos &= IPTOS_RT_MASK;
2282
	hash = rt_hash(daddr, saddr, iif, rt_genid(net));
L
Linus Torvalds 已提交
2283 2284

	for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2285
	     rth = rcu_dereference(rth->dst.rt_next)) {
2286 2287
		if ((((__force u32)rth->fl.fl4_dst ^ (__force u32)daddr) |
		     ((__force u32)rth->fl.fl4_src ^ (__force u32)saddr) |
2288 2289 2290
		     (rth->fl.iif ^ iif) |
		     rth->fl.oif |
		     (rth->fl.fl4_tos ^ tos)) == 0 &&
2291
		    rth->fl.mark == skb->mark &&
2292
		    net_eq(dev_net(rth->dst.dev), net) &&
2293
		    !rt_is_expired(rth)) {
2294
			if (noref) {
2295 2296
				dst_use_noref(&rth->dst, jiffies);
				skb_dst_set_noref(skb, &rth->dst);
2297
			} else {
2298 2299
				dst_use(&rth->dst, jiffies);
				skb_dst_set(skb, &rth->dst);
2300
			}
L
Linus Torvalds 已提交
2301 2302 2303 2304 2305 2306 2307
			RT_CACHE_STAT_INC(in_hit);
			rcu_read_unlock();
			return 0;
		}
		RT_CACHE_STAT_INC(in_hlist_search);
	}

2308
skip_cache:
L
Linus Torvalds 已提交
2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
	/* Multicast recognition logic is moved from route cache to here.
	   The problem was that too many Ethernet cards have broken/missing
	   hardware multicast filters :-( As result the host on multicasting
	   network acquires a lot of useless route cache entries, sort of
	   SDR messages from all the world. Now we try to get rid of them.
	   Really, provided software IP multicast filter is organized
	   reasonably (at least, hashed), it does not result in a slowdown
	   comparing with route cache reject entries.
	   Note, that multicast routers are not affected, because
	   route cache entry is created eventually.
	 */
2320
	if (ipv4_is_multicast(daddr)) {
2321
		struct in_device *in_dev = __in_dev_get_rcu(dev);
L
Linus Torvalds 已提交
2322

2323
		if (in_dev) {
L
Linus Torvalds 已提交
2324
			int our = ip_check_mc(in_dev, daddr, saddr,
2325
					      ip_hdr(skb)->protocol);
L
Linus Torvalds 已提交
2326 2327
			if (our
#ifdef CONFIG_IP_MROUTE
2328 2329 2330
				||
			    (!ipv4_is_local_multicast(daddr) &&
			     IN_DEV_MFORWARD(in_dev))
L
Linus Torvalds 已提交
2331
#endif
2332
			   ) {
2333 2334
				int res = ip_route_input_mc(skb, daddr, saddr,
							    tos, dev, our);
L
Linus Torvalds 已提交
2335
				rcu_read_unlock();
2336
				return res;
L
Linus Torvalds 已提交
2337 2338 2339 2340 2341
			}
		}
		rcu_read_unlock();
		return -EINVAL;
	}
2342 2343 2344
	res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
	rcu_read_unlock();
	return res;
L
Linus Torvalds 已提交
2345
}
2346
EXPORT_SYMBOL(ip_route_input_common);
L
Linus Torvalds 已提交
2347

E
Eric Dumazet 已提交
2348
/* called with rcu_read_lock() */
S
Stephen Hemminger 已提交
2349 2350 2351 2352 2353 2354
static int __mkroute_output(struct rtable **result,
			    struct fib_result *res,
			    const struct flowi *fl,
			    const struct flowi *oldflp,
			    struct net_device *dev_out,
			    unsigned flags)
L
Linus Torvalds 已提交
2355 2356 2357 2358 2359
{
	struct rtable *rth;
	struct in_device *in_dev;
	u32 tos = RT_FL_TOS(oldflp);

E
Eric Dumazet 已提交
2360
	if (ipv4_is_loopback(fl->fl4_src) && !(dev_out->flags & IFF_LOOPBACK))
L
Linus Torvalds 已提交
2361 2362
		return -EINVAL;

2363
	if (ipv4_is_lbcast(fl->fl4_dst))
L
Linus Torvalds 已提交
2364
		res->type = RTN_BROADCAST;
2365
	else if (ipv4_is_multicast(fl->fl4_dst))
L
Linus Torvalds 已提交
2366
		res->type = RTN_MULTICAST;
2367
	else if (ipv4_is_zeronet(fl->fl4_dst))
L
Linus Torvalds 已提交
2368 2369 2370 2371 2372
		return -EINVAL;

	if (dev_out->flags & IFF_LOOPBACK)
		flags |= RTCF_LOCAL;

E
Eric Dumazet 已提交
2373
	in_dev = __in_dev_get_rcu(dev_out);
E
Eric Dumazet 已提交
2374
	if (!in_dev)
L
Linus Torvalds 已提交
2375
		return -EINVAL;
E
Eric Dumazet 已提交
2376

L
Linus Torvalds 已提交
2377 2378
	if (res->type == RTN_BROADCAST) {
		flags |= RTCF_BROADCAST | RTCF_LOCAL;
E
Eric Dumazet 已提交
2379
		res->fi = NULL;
L
Linus Torvalds 已提交
2380
	} else if (res->type == RTN_MULTICAST) {
E
Eric Dumazet 已提交
2381
		flags |= RTCF_MULTICAST | RTCF_LOCAL;
2382
		if (!ip_check_mc(in_dev, oldflp->fl4_dst, oldflp->fl4_src,
L
Linus Torvalds 已提交
2383 2384 2385
				 oldflp->proto))
			flags &= ~RTCF_LOCAL;
		/* If multicast route do not exist use
E
Eric Dumazet 已提交
2386 2387
		 * default one, but do not gateway in this case.
		 * Yes, it is hack.
L
Linus Torvalds 已提交
2388
		 */
E
Eric Dumazet 已提交
2389
		if (res->fi && res->prefixlen < 4)
L
Linus Torvalds 已提交
2390 2391 2392 2393 2394
			res->fi = NULL;
	}


	rth = dst_alloc(&ipv4_dst_ops);
2395
	if (!rth)
E
Eric Dumazet 已提交
2396
		return -ENOBUFS;
2397

2398 2399
	atomic_set(&rth->dst.__refcnt, 1);
	rth->dst.flags= DST_HOST;
2400
	if (IN_DEV_CONF_GET(in_dev, NOXFRM))
2401
		rth->dst.flags |= DST_NOXFRM;
2402
	if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2403
		rth->dst.flags |= DST_NOPOLICY;
L
Linus Torvalds 已提交
2404 2405 2406 2407 2408

	rth->fl.fl4_dst	= oldflp->fl4_dst;
	rth->fl.fl4_tos	= tos;
	rth->fl.fl4_src	= oldflp->fl4_src;
	rth->fl.oif	= oldflp->oif;
2409
	rth->fl.mark    = oldflp->mark;
L
Linus Torvalds 已提交
2410 2411 2412
	rth->rt_dst	= fl->fl4_dst;
	rth->rt_src	= fl->fl4_src;
	rth->rt_iif	= oldflp->oif ? : dev_out->ifindex;
2413
	/* get references to the devices that are to be hold by the routing
L
Linus Torvalds 已提交
2414
	   cache entry */
2415
	rth->dst.dev	= dev_out;
L
Linus Torvalds 已提交
2416 2417 2418 2419
	dev_hold(dev_out);
	rth->rt_gateway = fl->fl4_dst;
	rth->rt_spec_dst= fl->fl4_src;

2420 2421
	rth->dst.output=ip_output;
	rth->dst.obsolete = -1;
2422
	rth->rt_genid = rt_genid(dev_net(dev_out));
L
Linus Torvalds 已提交
2423 2424 2425 2426

	RT_CACHE_STAT_INC(out_slow_tot);

	if (flags & RTCF_LOCAL) {
2427
		rth->dst.input = ip_local_deliver;
L
Linus Torvalds 已提交
2428 2429 2430 2431
		rth->rt_spec_dst = fl->fl4_dst;
	}
	if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
		rth->rt_spec_dst = fl->fl4_src;
2432
		if (flags & RTCF_LOCAL &&
L
Linus Torvalds 已提交
2433
		    !(dev_out->flags & IFF_LOOPBACK)) {
2434
			rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2435 2436 2437 2438 2439
			RT_CACHE_STAT_INC(out_slow_mc);
		}
#ifdef CONFIG_IP_MROUTE
		if (res->type == RTN_MULTICAST) {
			if (IN_DEV_MFORWARD(in_dev) &&
2440
			    !ipv4_is_local_multicast(oldflp->fl4_dst)) {
2441 2442
				rth->dst.input = ip_mr_input;
				rth->dst.output = ip_mc_output;
L
Linus Torvalds 已提交
2443 2444 2445 2446 2447 2448 2449 2450 2451
			}
		}
#endif
	}

	rt_set_nexthop(rth, res, 0);

	rth->rt_flags = flags;
	*result = rth;
E
Eric Dumazet 已提交
2452
	return 0;
L
Linus Torvalds 已提交
2453 2454
}

E
Eric Dumazet 已提交
2455
/* called with rcu_read_lock() */
S
Stephen Hemminger 已提交
2456 2457 2458 2459 2460 2461
static int ip_mkroute_output(struct rtable **rp,
			     struct fib_result *res,
			     const struct flowi *fl,
			     const struct flowi *oldflp,
			     struct net_device *dev_out,
			     unsigned flags)
L
Linus Torvalds 已提交
2462
{
C
Chuck Short 已提交
2463
	struct rtable *rth = NULL;
L
Linus Torvalds 已提交
2464 2465 2466
	int err = __mkroute_output(&rth, res, fl, oldflp, dev_out, flags);
	unsigned hash;
	if (err == 0) {
2467
		hash = rt_hash(oldflp->fl4_dst, oldflp->fl4_src, oldflp->oif,
2468
			       rt_genid(dev_net(dev_out)));
2469
		err = rt_intern_hash(hash, rth, rp, NULL, oldflp->oif);
L
Linus Torvalds 已提交
2470
	}
2471

L
Linus Torvalds 已提交
2472 2473 2474 2475 2476
	return err;
}

/*
 * Major route resolver routine.
2477
 * called with rcu_read_lock();
L
Linus Torvalds 已提交
2478 2479
 */

2480 2481
static int ip_route_output_slow(struct net *net, struct rtable **rp,
				const struct flowi *oldflp)
L
Linus Torvalds 已提交
2482 2483
{
	u32 tos	= RT_FL_TOS(oldflp);
2484 2485 2486 2487 2488
	struct flowi fl = { .fl4_dst = oldflp->fl4_dst,
			    .fl4_src = oldflp->fl4_src,
			    .fl4_tos = tos & IPTOS_RT_MASK,
			    .fl4_scope = ((tos & RTO_ONLINK) ?
					  RT_SCOPE_LINK : RT_SCOPE_UNIVERSE),
2489
			    .mark = oldflp->mark,
2490
			    .iif = net->loopback_dev->ifindex,
L
Linus Torvalds 已提交
2491 2492
			    .oif = oldflp->oif };
	struct fib_result res;
2493
	unsigned int flags = 0;
L
Linus Torvalds 已提交
2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504
	struct net_device *dev_out = NULL;
	int err;


	res.fi		= NULL;
#ifdef CONFIG_IP_MULTIPLE_TABLES
	res.r		= NULL;
#endif

	if (oldflp->fl4_src) {
		err = -EINVAL;
2505
		if (ipv4_is_multicast(oldflp->fl4_src) ||
2506
		    ipv4_is_lbcast(oldflp->fl4_src) ||
2507
		    ipv4_is_zeronet(oldflp->fl4_src))
L
Linus Torvalds 已提交
2508 2509 2510 2511
			goto out;

		/* I removed check for oif == dev_out->oif here.
		   It was wrong for two reasons:
2512 2513
		   1. ip_dev_find(net, saddr) can return wrong iface, if saddr
		      is assigned to multiple interfaces.
L
Linus Torvalds 已提交
2514 2515 2516 2517
		   2. Moreover, we are allowed to send packets with saddr
		      of another iface. --ANK
		 */

2518 2519
		if (oldflp->oif == 0 &&
		    (ipv4_is_multicast(oldflp->fl4_dst) ||
2520
		     ipv4_is_lbcast(oldflp->fl4_dst))) {
2521
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2522
			dev_out = __ip_dev_find(net, oldflp->fl4_src, false);
2523 2524 2525
			if (dev_out == NULL)
				goto out;

L
Linus Torvalds 已提交
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543
			/* Special hack: user can direct multicasts
			   and limited broadcast via necessary interface
			   without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
			   This hack is not just for fun, it allows
			   vic,vat and friends to work.
			   They bind socket to loopback, set ttl to zero
			   and expect that it will work.
			   From the viewpoint of routing cache they are broken,
			   because we are not allowed to build multicast path
			   with loopback source addr (look, routing cache
			   cannot know, that ttl is zero, so that packet
			   will not leave this host and route is valid).
			   Luckily, this hack is good workaround.
			 */

			fl.oif = dev_out->ifindex;
			goto make_route;
		}
2544 2545 2546

		if (!(oldflp->flags & FLOWI_FLAG_ANYSRC)) {
			/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2547
			if (!__ip_dev_find(net, oldflp->fl4_src, false))
2548 2549
				goto out;
		}
L
Linus Torvalds 已提交
2550 2551 2552 2553
	}


	if (oldflp->oif) {
2554
		dev_out = dev_get_by_index_rcu(net, oldflp->oif);
L
Linus Torvalds 已提交
2555 2556 2557
		err = -ENODEV;
		if (dev_out == NULL)
			goto out;
2558 2559

		/* RACE: Check return value of inet_select_addr instead. */
2560
		if (rcu_dereference(dev_out->ip_ptr) == NULL)
L
Linus Torvalds 已提交
2561 2562
			goto out;	/* Wrong error code */

2563
		if (ipv4_is_local_multicast(oldflp->fl4_dst) ||
2564
		    ipv4_is_lbcast(oldflp->fl4_dst)) {
L
Linus Torvalds 已提交
2565 2566 2567 2568 2569 2570
			if (!fl.fl4_src)
				fl.fl4_src = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
			goto make_route;
		}
		if (!fl.fl4_src) {
2571
			if (ipv4_is_multicast(oldflp->fl4_dst))
L
Linus Torvalds 已提交
2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583
				fl.fl4_src = inet_select_addr(dev_out, 0,
							      fl.fl4_scope);
			else if (!oldflp->fl4_dst)
				fl.fl4_src = inet_select_addr(dev_out, 0,
							      RT_SCOPE_HOST);
		}
	}

	if (!fl.fl4_dst) {
		fl.fl4_dst = fl.fl4_src;
		if (!fl.fl4_dst)
			fl.fl4_dst = fl.fl4_src = htonl(INADDR_LOOPBACK);
2584 2585
		dev_out = net->loopback_dev;
		fl.oif = net->loopback_dev->ifindex;
L
Linus Torvalds 已提交
2586 2587 2588 2589 2590
		res.type = RTN_LOCAL;
		flags |= RTCF_LOCAL;
		goto make_route;
	}

2591
	if (fib_lookup(net, &fl, &res)) {
L
Linus Torvalds 已提交
2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624
		res.fi = NULL;
		if (oldflp->oif) {
			/* Apparently, routing tables are wrong. Assume,
			   that the destination is on link.

			   WHY? DW.
			   Because we are allowed to send to iface
			   even if it has NO routes and NO assigned
			   addresses. When oif is specified, routing
			   tables are looked up with only one purpose:
			   to catch if destination is gatewayed, rather than
			   direct. Moreover, if MSG_DONTROUTE is set,
			   we send packet, ignoring both routing tables
			   and ifaddr state. --ANK


			   We could make it even if oif is unknown,
			   likely IPv6, but we do not.
			 */

			if (fl.fl4_src == 0)
				fl.fl4_src = inet_select_addr(dev_out, 0,
							      RT_SCOPE_LINK);
			res.type = RTN_UNICAST;
			goto make_route;
		}
		err = -ENETUNREACH;
		goto out;
	}

	if (res.type == RTN_LOCAL) {
		if (!fl.fl4_src)
			fl.fl4_src = fl.fl4_dst;
2625
		dev_out = net->loopback_dev;
L
Linus Torvalds 已提交
2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
		fl.oif = dev_out->ifindex;
		res.fi = NULL;
		flags |= RTCF_LOCAL;
		goto make_route;
	}

#ifdef CONFIG_IP_ROUTE_MULTIPATH
	if (res.fi->fib_nhs > 1 && fl.oif == 0)
		fib_select_multipath(&fl, &res);
	else
#endif
	if (!res.prefixlen && res.type == RTN_UNICAST && !fl.oif)
2638
		fib_select_default(net, &fl, &res);
L
Linus Torvalds 已提交
2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652

	if (!fl.fl4_src)
		fl.fl4_src = FIB_RES_PREFSRC(res);

	dev_out = FIB_RES_DEV(res);
	fl.oif = dev_out->ifindex;


make_route:
	err = ip_mkroute_output(rp, &res, &fl, oldflp, dev_out, flags);

out:	return err;
}

2653 2654
int __ip_route_output_key(struct net *net, struct rtable **rp,
			  const struct flowi *flp)
L
Linus Torvalds 已提交
2655
{
2656 2657
	unsigned int hash;
	int res;
L
Linus Torvalds 已提交
2658 2659
	struct rtable *rth;

2660 2661 2662
	if (!rt_caching(net))
		goto slow_output;

2663
	hash = rt_hash(flp->fl4_dst, flp->fl4_src, flp->oif, rt_genid(net));
L
Linus Torvalds 已提交
2664 2665

	rcu_read_lock_bh();
2666
	for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2667
		rth = rcu_dereference_bh(rth->dst.rt_next)) {
L
Linus Torvalds 已提交
2668 2669
		if (rth->fl.fl4_dst == flp->fl4_dst &&
		    rth->fl.fl4_src == flp->fl4_src &&
2670
		    rt_is_output_route(rth) &&
L
Linus Torvalds 已提交
2671
		    rth->fl.oif == flp->oif &&
2672
		    rth->fl.mark == flp->mark &&
L
Linus Torvalds 已提交
2673
		    !((rth->fl.fl4_tos ^ flp->fl4_tos) &
2674
			    (IPTOS_RT_MASK | RTO_ONLINK)) &&
2675
		    net_eq(dev_net(rth->dst.dev), net) &&
2676
		    !rt_is_expired(rth)) {
2677
			dst_use(&rth->dst, jiffies);
L
Linus Torvalds 已提交
2678 2679 2680 2681 2682 2683 2684 2685 2686
			RT_CACHE_STAT_INC(out_hit);
			rcu_read_unlock_bh();
			*rp = rth;
			return 0;
		}
		RT_CACHE_STAT_INC(out_hlist_search);
	}
	rcu_read_unlock_bh();

2687
slow_output:
2688 2689 2690 2691
	rcu_read_lock();
	res = ip_route_output_slow(net, rp, flp);
	rcu_read_unlock();
	return res;
L
Linus Torvalds 已提交
2692
}
2693 2694
EXPORT_SYMBOL_GPL(__ip_route_output_key);

2695 2696 2697 2698 2699
static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
{
	return NULL;
}

2700 2701 2702 2703 2704 2705
static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
{
}

static struct dst_ops ipv4_dst_blackhole_ops = {
	.family			=	AF_INET,
2706
	.protocol		=	cpu_to_be16(ETH_P_IP),
2707
	.destroy		=	ipv4_dst_destroy,
2708
	.check			=	ipv4_blackhole_dst_check,
2709 2710 2711 2712
	.update_pmtu		=	ipv4_rt_blackhole_update_pmtu,
};


2713
static int ipv4_dst_blackhole(struct net *net, struct rtable **rp, struct flowi *flp)
2714 2715 2716 2717 2718 2719
{
	struct rtable *ort = *rp;
	struct rtable *rt = (struct rtable *)
		dst_alloc(&ipv4_dst_blackhole_ops);

	if (rt) {
2720
		struct dst_entry *new = &rt->dst;
2721 2722 2723

		atomic_set(&new->__refcnt, 1);
		new->__use = 1;
2724 2725
		new->input = dst_discard;
		new->output = dst_discard;
2726
		dst_copy_metrics(new, &ort->dst);
2727

2728
		new->dev = ort->dst.dev;
2729 2730 2731 2732 2733
		if (new->dev)
			dev_hold(new->dev);

		rt->fl = ort->fl;

2734
		rt->rt_genid = rt_genid(net);
2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748
		rt->rt_flags = ort->rt_flags;
		rt->rt_type = ort->rt_type;
		rt->rt_dst = ort->rt_dst;
		rt->rt_src = ort->rt_src;
		rt->rt_iif = ort->rt_iif;
		rt->rt_gateway = ort->rt_gateway;
		rt->rt_spec_dst = ort->rt_spec_dst;
		rt->peer = ort->peer;
		if (rt->peer)
			atomic_inc(&rt->peer->refcnt);

		dst_free(new);
	}

2749
	dst_release(&(*rp)->dst);
2750
	*rp = rt;
E
Eric Dumazet 已提交
2751
	return rt ? 0 : -ENOMEM;
2752 2753
}

2754 2755
int ip_route_output_flow(struct net *net, struct rtable **rp, struct flowi *flp,
			 struct sock *sk, int flags)
L
Linus Torvalds 已提交
2756 2757 2758
{
	int err;

2759
	if ((err = __ip_route_output_key(net, rp, flp)) != 0)
L
Linus Torvalds 已提交
2760 2761 2762 2763 2764 2765 2766
		return err;

	if (flp->proto) {
		if (!flp->fl4_src)
			flp->fl4_src = (*rp)->rt_src;
		if (!flp->fl4_dst)
			flp->fl4_dst = (*rp)->rt_dst;
A
Alexey Dobriyan 已提交
2767
		err = __xfrm_lookup(net, (struct dst_entry **)rp, flp, sk,
2768
				    flags ? XFRM_LOOKUP_WAIT : 0);
2769
		if (err == -EREMOTE)
2770
			err = ipv4_dst_blackhole(net, rp, flp);
2771 2772

		return err;
L
Linus Torvalds 已提交
2773 2774 2775 2776
	}

	return 0;
}
2777 2778
EXPORT_SYMBOL_GPL(ip_route_output_flow);

2779
int ip_route_output_key(struct net *net, struct rtable **rp, struct flowi *flp)
L
Linus Torvalds 已提交
2780
{
2781
	return ip_route_output_flow(net, rp, flp, NULL, 0);
L
Linus Torvalds 已提交
2782
}
E
Eric Dumazet 已提交
2783
EXPORT_SYMBOL(ip_route_output_key);
L
Linus Torvalds 已提交
2784

2785 2786
static int rt_fill_info(struct net *net,
			struct sk_buff *skb, u32 pid, u32 seq, int event,
2787
			int nowait, unsigned int flags)
L
Linus Torvalds 已提交
2788
{
E
Eric Dumazet 已提交
2789
	struct rtable *rt = skb_rtable(skb);
L
Linus Torvalds 已提交
2790
	struct rtmsg *r;
2791
	struct nlmsghdr *nlh;
2792 2793
	long expires;
	u32 id = 0, ts = 0, tsage = 0, error;
2794 2795 2796

	nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
	if (nlh == NULL)
2797
		return -EMSGSIZE;
2798 2799

	r = nlmsg_data(nlh);
L
Linus Torvalds 已提交
2800 2801 2802 2803 2804
	r->rtm_family	 = AF_INET;
	r->rtm_dst_len	= 32;
	r->rtm_src_len	= 0;
	r->rtm_tos	= rt->fl.fl4_tos;
	r->rtm_table	= RT_TABLE_MAIN;
2805
	NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
L
Linus Torvalds 已提交
2806 2807 2808 2809 2810 2811
	r->rtm_type	= rt->rt_type;
	r->rtm_scope	= RT_SCOPE_UNIVERSE;
	r->rtm_protocol = RTPROT_UNSPEC;
	r->rtm_flags	= (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
	if (rt->rt_flags & RTCF_NOTIFY)
		r->rtm_flags |= RTM_F_NOTIFY;
2812

2813
	NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
2814

L
Linus Torvalds 已提交
2815 2816
	if (rt->fl.fl4_src) {
		r->rtm_src_len = 32;
2817
		NLA_PUT_BE32(skb, RTA_SRC, rt->fl.fl4_src);
L
Linus Torvalds 已提交
2818
	}
2819 2820
	if (rt->dst.dev)
		NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
L
Linus Torvalds 已提交
2821
#ifdef CONFIG_NET_CLS_ROUTE
2822 2823
	if (rt->dst.tclassid)
		NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
L
Linus Torvalds 已提交
2824
#endif
2825
	if (rt_is_input_route(rt))
2826
		NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
L
Linus Torvalds 已提交
2827
	else if (rt->rt_src != rt->fl.fl4_src)
2828
		NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
2829

L
Linus Torvalds 已提交
2830
	if (rt->rt_dst != rt->rt_gateway)
2831
		NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
2832

2833
	if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
2834 2835
		goto nla_put_failure;

E
Eric Dumazet 已提交
2836 2837 2838
	if (rt->fl.mark)
		NLA_PUT_BE32(skb, RTA_MARK, rt->fl.mark);

2839 2840
	error = rt->dst.error;
	expires = rt->dst.expires ? rt->dst.expires - jiffies : 0;
L
Linus Torvalds 已提交
2841
	if (rt->peer) {
2842
		inet_peer_refcheck(rt->peer);
E
Eric Dumazet 已提交
2843
		id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
L
Linus Torvalds 已提交
2844
		if (rt->peer->tcp_ts_stamp) {
2845
			ts = rt->peer->tcp_ts;
2846
			tsage = get_seconds() - rt->peer->tcp_ts_stamp;
L
Linus Torvalds 已提交
2847 2848
		}
	}
2849

2850
	if (rt_is_input_route(rt)) {
L
Linus Torvalds 已提交
2851
#ifdef CONFIG_IP_MROUTE
2852
		__be32 dst = rt->rt_dst;
L
Linus Torvalds 已提交
2853

2854
		if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2855 2856
		    IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
			int err = ipmr_get_route(net, skb, r, nowait);
L
Linus Torvalds 已提交
2857 2858 2859 2860
			if (err <= 0) {
				if (!nowait) {
					if (err == 0)
						return 0;
2861
					goto nla_put_failure;
L
Linus Torvalds 已提交
2862 2863
				} else {
					if (err == -EMSGSIZE)
2864
						goto nla_put_failure;
2865
					error = err;
L
Linus Torvalds 已提交
2866 2867 2868 2869
				}
			}
		} else
#endif
2870
			NLA_PUT_U32(skb, RTA_IIF, rt->fl.iif);
L
Linus Torvalds 已提交
2871 2872
	}

2873
	if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2874 2875
			       expires, error) < 0)
		goto nla_put_failure;
2876 2877

	return nlmsg_end(skb, nlh);
L
Linus Torvalds 已提交
2878

2879
nla_put_failure:
2880 2881
	nlmsg_cancel(skb, nlh);
	return -EMSGSIZE;
L
Linus Torvalds 已提交
2882 2883
}

2884
static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
L
Linus Torvalds 已提交
2885
{
2886
	struct net *net = sock_net(in_skb->sk);
2887 2888
	struct rtmsg *rtm;
	struct nlattr *tb[RTA_MAX+1];
L
Linus Torvalds 已提交
2889
	struct rtable *rt = NULL;
A
Al Viro 已提交
2890 2891 2892
	__be32 dst = 0;
	__be32 src = 0;
	u32 iif;
2893
	int err;
E
Eric Dumazet 已提交
2894
	int mark;
L
Linus Torvalds 已提交
2895 2896
	struct sk_buff *skb;

2897 2898 2899 2900 2901 2902
	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
	if (err < 0)
		goto errout;

	rtm = nlmsg_data(nlh);

L
Linus Torvalds 已提交
2903
	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2904 2905 2906 2907
	if (skb == NULL) {
		err = -ENOBUFS;
		goto errout;
	}
L
Linus Torvalds 已提交
2908 2909 2910 2911

	/* Reserve room for dummy headers, this skb can pass
	   through good chunk of routing engine.
	 */
2912
	skb_reset_mac_header(skb);
2913
	skb_reset_network_header(skb);
2914 2915

	/* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2916
	ip_hdr(skb)->protocol = IPPROTO_ICMP;
L
Linus Torvalds 已提交
2917 2918
	skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));

2919 2920
	src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
	dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2921
	iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
E
Eric Dumazet 已提交
2922
	mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
L
Linus Torvalds 已提交
2923 2924

	if (iif) {
2925 2926
		struct net_device *dev;

2927
		dev = __dev_get_by_index(net, iif);
2928 2929 2930 2931 2932
		if (dev == NULL) {
			err = -ENODEV;
			goto errout_free;
		}

L
Linus Torvalds 已提交
2933 2934
		skb->protocol	= htons(ETH_P_IP);
		skb->dev	= dev;
E
Eric Dumazet 已提交
2935
		skb->mark	= mark;
L
Linus Torvalds 已提交
2936 2937 2938
		local_bh_disable();
		err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
		local_bh_enable();
2939

E
Eric Dumazet 已提交
2940
		rt = skb_rtable(skb);
2941 2942
		if (err == 0 && rt->dst.error)
			err = -rt->dst.error;
L
Linus Torvalds 已提交
2943
	} else {
2944
		struct flowi fl = {
2945 2946 2947
			.fl4_dst = dst,
			.fl4_src = src,
			.fl4_tos = rtm->rtm_tos,
2948
			.oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
E
Eric Dumazet 已提交
2949
			.mark = mark,
2950
		};
2951
		err = ip_route_output_key(net, &rt, &fl);
L
Linus Torvalds 已提交
2952
	}
2953

L
Linus Torvalds 已提交
2954
	if (err)
2955
		goto errout_free;
L
Linus Torvalds 已提交
2956

2957
	skb_dst_set(skb, &rt->dst);
L
Linus Torvalds 已提交
2958 2959 2960
	if (rtm->rtm_flags & RTM_F_NOTIFY)
		rt->rt_flags |= RTCF_NOTIFY;

2961
	err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
2962
			   RTM_NEWROUTE, 0, 0);
2963 2964
	if (err <= 0)
		goto errout_free;
L
Linus Torvalds 已提交
2965

2966
	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
2967
errout:
2968
	return err;
L
Linus Torvalds 已提交
2969

2970
errout_free:
L
Linus Torvalds 已提交
2971
	kfree_skb(skb);
2972
	goto errout;
L
Linus Torvalds 已提交
2973 2974 2975 2976 2977 2978 2979
}

int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
{
	struct rtable *rt;
	int h, s_h;
	int idx, s_idx;
2980 2981
	struct net *net;

2982
	net = sock_net(skb->sk);
L
Linus Torvalds 已提交
2983 2984

	s_h = cb->args[0];
2985 2986
	if (s_h < 0)
		s_h = 0;
L
Linus Torvalds 已提交
2987
	s_idx = idx = cb->args[1];
2988 2989 2990
	for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
		if (!rt_hash_table[h].chain)
			continue;
L
Linus Torvalds 已提交
2991
		rcu_read_lock_bh();
2992
		for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
2993 2994
		     rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
			if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
L
Linus Torvalds 已提交
2995
				continue;
2996
			if (rt_is_expired(rt))
2997
				continue;
2998
			skb_dst_set_noref(skb, &rt->dst);
2999
			if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
3000
					 cb->nlh->nlmsg_seq, RTM_NEWROUTE,
3001
					 1, NLM_F_MULTI) <= 0) {
E
Eric Dumazet 已提交
3002
				skb_dst_drop(skb);
L
Linus Torvalds 已提交
3003 3004 3005
				rcu_read_unlock_bh();
				goto done;
			}
E
Eric Dumazet 已提交
3006
			skb_dst_drop(skb);
L
Linus Torvalds 已提交
3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018
		}
		rcu_read_unlock_bh();
	}

done:
	cb->args[0] = h;
	cb->args[1] = idx;
	return skb->len;
}

void ip_rt_multicast_event(struct in_device *in_dev)
{
3019
	rt_cache_flush(dev_net(in_dev->dev), 0);
L
Linus Torvalds 已提交
3020 3021 3022
}

#ifdef CONFIG_SYSCTL
3023
static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3024
					void __user *buffer,
L
Linus Torvalds 已提交
3025 3026 3027
					size_t *lenp, loff_t *ppos)
{
	if (write) {
3028
		int flush_delay;
3029
		ctl_table ctl;
3030
		struct net *net;
3031

3032 3033
		memcpy(&ctl, __ctl, sizeof(ctl));
		ctl.data = &flush_delay;
3034
		proc_dointvec(&ctl, write, buffer, lenp, ppos);
3035

3036
		net = (struct net *)__ctl->extra1;
3037
		rt_cache_flush(net, flush_delay);
L
Linus Torvalds 已提交
3038
		return 0;
3039
	}
L
Linus Torvalds 已提交
3040 3041 3042 3043

	return -EINVAL;
}

A
Al Viro 已提交
3044
static ctl_table ipv4_route_table[] = {
L
Linus Torvalds 已提交
3045 3046 3047 3048 3049
	{
		.procname	= "gc_thresh",
		.data		= &ipv4_dst_ops.gc_thresh,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3050
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3051 3052 3053 3054 3055 3056
	},
	{
		.procname	= "max_size",
		.data		= &ip_rt_max_size,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3057
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3058 3059 3060
	},
	{
		/*  Deprecated. Use gc_min_interval_ms */
3061

L
Linus Torvalds 已提交
3062 3063 3064 3065
		.procname	= "gc_min_interval",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3066
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3067 3068 3069 3070 3071 3072
	},
	{
		.procname	= "gc_min_interval_ms",
		.data		= &ip_rt_gc_min_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3073
		.proc_handler	= proc_dointvec_ms_jiffies,
L
Linus Torvalds 已提交
3074 3075 3076 3077 3078 3079
	},
	{
		.procname	= "gc_timeout",
		.data		= &ip_rt_gc_timeout,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3080
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3081 3082 3083 3084 3085 3086
	},
	{
		.procname	= "gc_interval",
		.data		= &ip_rt_gc_interval,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3087
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3088 3089 3090 3091 3092 3093
	},
	{
		.procname	= "redirect_load",
		.data		= &ip_rt_redirect_load,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3094
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3095 3096 3097 3098 3099 3100
	},
	{
		.procname	= "redirect_number",
		.data		= &ip_rt_redirect_number,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3101
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3102 3103 3104 3105 3106 3107
	},
	{
		.procname	= "redirect_silence",
		.data		= &ip_rt_redirect_silence,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3108
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3109 3110 3111 3112 3113 3114
	},
	{
		.procname	= "error_cost",
		.data		= &ip_rt_error_cost,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3115
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3116 3117 3118 3119 3120 3121
	},
	{
		.procname	= "error_burst",
		.data		= &ip_rt_error_burst,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3122
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3123 3124 3125 3126 3127 3128
	},
	{
		.procname	= "gc_elasticity",
		.data		= &ip_rt_gc_elasticity,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3129
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3130 3131 3132 3133 3134 3135
	},
	{
		.procname	= "mtu_expires",
		.data		= &ip_rt_mtu_expires,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3136
		.proc_handler	= proc_dointvec_jiffies,
L
Linus Torvalds 已提交
3137 3138 3139 3140 3141 3142
	},
	{
		.procname	= "min_pmtu",
		.data		= &ip_rt_min_pmtu,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3143
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3144 3145 3146 3147 3148 3149
	},
	{
		.procname	= "min_adv_mss",
		.data		= &ip_rt_min_advmss,
		.maxlen		= sizeof(int),
		.mode		= 0644,
A
Alexey Dobriyan 已提交
3150
		.proc_handler	= proc_dointvec,
L
Linus Torvalds 已提交
3151
	},
3152
	{ }
L
Linus Torvalds 已提交
3153
};
3154

A
Al Viro 已提交
3155 3156 3157 3158
static struct ctl_table empty[1];

static struct ctl_table ipv4_skeleton[] =
{
3159
	{ .procname = "route", 
H
Hugh Dickins 已提交
3160
	  .mode = 0555, .child = ipv4_route_table},
3161
	{ .procname = "neigh", 
H
Hugh Dickins 已提交
3162
	  .mode = 0555, .child = empty},
A
Al Viro 已提交
3163 3164 3165 3166
	{ }
};

static __net_initdata struct ctl_path ipv4_path[] = {
3167 3168
	{ .procname = "net", },
	{ .procname = "ipv4", },
3169 3170 3171 3172 3173 3174 3175 3176
	{ },
};

static struct ctl_table ipv4_route_flush_table[] = {
	{
		.procname	= "flush",
		.maxlen		= sizeof(int),
		.mode		= 0200,
A
Alexey Dobriyan 已提交
3177
		.proc_handler	= ipv4_sysctl_rtcache_flush,
3178
	},
3179
	{ },
3180 3181
};

A
Al Viro 已提交
3182
static __net_initdata struct ctl_path ipv4_route_path[] = {
3183 3184 3185
	{ .procname = "net", },
	{ .procname = "ipv4", },
	{ .procname = "route", },
A
Al Viro 已提交
3186 3187 3188
	{ },
};

3189 3190 3191 3192 3193
static __net_init int sysctl_route_net_init(struct net *net)
{
	struct ctl_table *tbl;

	tbl = ipv4_route_flush_table;
O
Octavian Purdila 已提交
3194
	if (!net_eq(net, &init_net)) {
3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227
		tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
		if (tbl == NULL)
			goto err_dup;
	}
	tbl[0].extra1 = net;

	net->ipv4.route_hdr =
		register_net_sysctl_table(net, ipv4_route_path, tbl);
	if (net->ipv4.route_hdr == NULL)
		goto err_reg;
	return 0;

err_reg:
	if (tbl != ipv4_route_flush_table)
		kfree(tbl);
err_dup:
	return -ENOMEM;
}

static __net_exit void sysctl_route_net_exit(struct net *net)
{
	struct ctl_table *tbl;

	tbl = net->ipv4.route_hdr->ctl_table_arg;
	unregister_net_sysctl_table(net->ipv4.route_hdr);
	BUG_ON(tbl == ipv4_route_flush_table);
	kfree(tbl);
}

static __net_initdata struct pernet_operations sysctl_route_ops = {
	.init = sysctl_route_net_init,
	.exit = sysctl_route_net_exit,
};
L
Linus Torvalds 已提交
3228 3229
#endif

3230
static __net_init int rt_genid_init(struct net *net)
3231
{
3232 3233
	get_random_bytes(&net->ipv4.rt_genid,
			 sizeof(net->ipv4.rt_genid));
3234 3235 3236
	return 0;
}

3237 3238
static __net_initdata struct pernet_operations rt_genid_ops = {
	.init = rt_genid_init,
3239 3240 3241
};


L
Linus Torvalds 已提交
3242
#ifdef CONFIG_NET_CLS_ROUTE
3243
struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
L
Linus Torvalds 已提交
3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257
#endif /* CONFIG_NET_CLS_ROUTE */

static __initdata unsigned long rhash_entries;
static int __init set_rhash_entries(char *str)
{
	if (!str)
		return 0;
	rhash_entries = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("rhash_entries=", set_rhash_entries);

int __init ip_rt_init(void)
{
3258
	int rc = 0;
L
Linus Torvalds 已提交
3259 3260

#ifdef CONFIG_NET_CLS_ROUTE
3261
	ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
L
Linus Torvalds 已提交
3262 3263 3264 3265
	if (!ip_rt_acct)
		panic("IP: failed to allocate ip_rt_acct\n");
#endif

A
Alexey Dobriyan 已提交
3266 3267
	ipv4_dst_ops.kmem_cachep =
		kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3268
				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3269

3270 3271
	ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;

3272 3273 3274 3275 3276 3277
	if (dst_entries_init(&ipv4_dst_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_ops counter\n");

	if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
		panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");

3278 3279 3280 3281
	rt_hash_table = (struct rt_hash_bucket *)
		alloc_large_system_hash("IP route cache",
					sizeof(struct rt_hash_bucket),
					rhash_entries,
3282
					(totalram_pages >= 128 * 1024) ?
3283
					15 : 17,
3284
					0,
3285 3286
					&rt_hash_log,
					&rt_hash_mask,
3287
					rhash_entries ? 0 : 512 * 1024);
3288 3289
	memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
	rt_hash_lock_init();
L
Linus Torvalds 已提交
3290 3291 3292 3293 3294 3295 3296 3297 3298 3299

	ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
	ip_rt_max_size = (rt_hash_mask + 1) * 16;

	devinet_init();
	ip_fib_init();

	/* All the timers, started at system startup tend
	   to synchronize. Perturb it a bit.
	 */
3300 3301
	INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
	expires_ljiffies = jiffies;
3302 3303
	schedule_delayed_work(&expires_work,
		net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
L
Linus Torvalds 已提交
3304

3305
	if (ip_rt_proc_init())
3306
		printk(KERN_ERR "Unable to create route proc files\n");
L
Linus Torvalds 已提交
3307 3308
#ifdef CONFIG_XFRM
	xfrm_init();
3309
	xfrm4_init(ip_rt_max_size);
L
Linus Torvalds 已提交
3310
#endif
3311 3312
	rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL);

3313 3314 3315
#ifdef CONFIG_SYSCTL
	register_pernet_subsys(&sysctl_route_ops);
#endif
3316
	register_pernet_subsys(&rt_genid_ops);
L
Linus Torvalds 已提交
3317 3318 3319
	return rc;
}

3320
#ifdef CONFIG_SYSCTL
A
Al Viro 已提交
3321 3322 3323 3324 3325 3326
/*
 * We really need to sanitize the damn ipv4 init order, then all
 * this nonsense will go away.
 */
void __init ip_static_sysctl_init(void)
{
A
Al Viro 已提交
3327
	register_sysctl_paths(ipv4_path, ipv4_skeleton);
A
Al Viro 已提交
3328
}
3329
#endif