route.c 132.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *	Linux INET6 implementation
 *	FIB front-end.
 *
 *	Authors:
6
 *	Pedro Roque		<roque@di.fc.ul.pt>
L
Linus Torvalds 已提交
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *
 *	This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

/*	Changes:
 *
 *	YOSHIFUJI Hideaki @USAGI
 *		reworked default router selection.
 *		- respect outgoing interface
 *		- select from (probably) reachable routers (i.e.
 *		routers in REACHABLE, STALE, DELAY or PROBE states).
 *		- always select the same router if it is (probably)
 *		reachable.  otherwise, round-robin the list.
23 24
 *	Ville Nuorvala
 *		Fixed routing subtrees.
L
Linus Torvalds 已提交
25 26
 */

27 28
#define pr_fmt(fmt) "IPv6: " fmt

29
#include <linux/capability.h>
L
Linus Torvalds 已提交
30
#include <linux/errno.h>
31
#include <linux/export.h>
L
Linus Torvalds 已提交
32 33 34 35 36 37 38 39
#include <linux/types.h>
#include <linux/times.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/route.h>
#include <linux/netdevice.h>
#include <linux/in6.h>
40
#include <linux/mroute6.h>
L
Linus Torvalds 已提交
41 42 43 44
#include <linux/init.h>
#include <linux/if_arp.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
45
#include <linux/nsproxy.h>
46
#include <linux/slab.h>
47
#include <linux/jhash.h>
48
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
49 50 51 52 53 54 55 56 57
#include <net/snmp.h>
#include <net/ipv6.h>
#include <net/ip6_fib.h>
#include <net/ip6_route.h>
#include <net/ndisc.h>
#include <net/addrconf.h>
#include <net/tcp.h>
#include <linux/rtnetlink.h>
#include <net/dst.h>
58
#include <net/dst_metadata.h>
L
Linus Torvalds 已提交
59
#include <net/xfrm.h>
60
#include <net/netevent.h>
61
#include <net/netlink.h>
62
#include <net/nexthop.h>
63
#include <net/lwtunnel.h>
64
#include <net/ip_tunnels.h>
D
David Ahern 已提交
65
#include <net/l3mdev.h>
66
#include <net/ip.h>
67
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
68 69 70 71 72

#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif

73 74 75 76 77 78 79
static int ip6_rt_type_to_error(u8 fib6_type);

#define CREATE_TRACE_POINTS
#include <trace/events/fib6.h>
EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup);
#undef CREATE_TRACE_POINTS

80
enum rt6_nud_state {
J
Jiri Benc 已提交
81 82 83
	RT6_NUD_FAIL_HARD = -3,
	RT6_NUD_FAIL_PROBE = -2,
	RT6_NUD_FAIL_DO_RR = -1,
84 85 86
	RT6_NUD_SUCCEED = 1
};

L
Linus Torvalds 已提交
87
static struct dst_entry	*ip6_dst_check(struct dst_entry *dst, u32 cookie);
88
static unsigned int	 ip6_default_advmss(const struct dst_entry *dst);
89
static unsigned int	 ip6_mtu(const struct dst_entry *dst);
L
Linus Torvalds 已提交
90 91 92 93
static struct dst_entry *ip6_negative_advice(struct dst_entry *);
static void		ip6_dst_destroy(struct dst_entry *);
static void		ip6_dst_ifdown(struct dst_entry *,
				       struct net_device *dev, int how);
94
static int		 ip6_dst_gc(struct dst_ops *ops);
L
Linus Torvalds 已提交
95 96

static int		ip6_pkt_discard(struct sk_buff *skb);
E
Eric W. Biederman 已提交
97
static int		ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
98
static int		ip6_pkt_prohibit(struct sk_buff *skb);
E
Eric W. Biederman 已提交
99
static int		ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb);
L
Linus Torvalds 已提交
100
static void		ip6_link_failure(struct sk_buff *skb);
101 102 103 104
static void		ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
					   struct sk_buff *skb, u32 mtu);
static void		rt6_do_redirect(struct dst_entry *dst, struct sock *sk,
					struct sk_buff *skb);
105 106
static int rt6_score_route(struct fib6_info *rt, int oif, int strict);
static size_t rt6_nlmsg_size(struct fib6_info *rt);
107
static int rt6_fill_node(struct net *net, struct sk_buff *skb,
108
			 struct fib6_info *rt, struct dst_entry *dst,
109
			 struct in6_addr *dest, struct in6_addr *src,
110 111
			 int iif, int type, u32 portid, u32 seq,
			 unsigned int flags);
112
static struct rt6_info *rt6_find_cached_rt(struct fib6_info *rt,
113 114
					   struct in6_addr *daddr,
					   struct in6_addr *saddr);
L
Linus Torvalds 已提交
115

116
#ifdef CONFIG_IPV6_ROUTE_INFO
117
static struct fib6_info *rt6_add_route_info(struct net *net,
118
					   const struct in6_addr *prefix, int prefixlen,
119 120
					   const struct in6_addr *gwaddr,
					   struct net_device *dev,
121
					   unsigned int pref);
122
static struct fib6_info *rt6_get_route_info(struct net *net,
123
					   const struct in6_addr *prefix, int prefixlen,
124 125
					   const struct in6_addr *gwaddr,
					   struct net_device *dev);
126 127
#endif

128 129 130 131 132 133 134
struct uncached_list {
	spinlock_t		lock;
	struct list_head	head;
};

static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list);

135
void rt6_uncached_list_add(struct rt6_info *rt)
136 137 138 139 140 141 142 143 144 145
{
	struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list);

	rt->rt6i_uncached_list = ul;

	spin_lock_bh(&ul->lock);
	list_add_tail(&rt->rt6i_uncached, &ul->head);
	spin_unlock_bh(&ul->lock);
}

146
void rt6_uncached_list_del(struct rt6_info *rt)
147 148 149
{
	if (!list_empty(&rt->rt6i_uncached)) {
		struct uncached_list *ul = rt->rt6i_uncached_list;
W
Wei Wang 已提交
150
		struct net *net = dev_net(rt->dst.dev);
151 152 153

		spin_lock_bh(&ul->lock);
		list_del(&rt->rt6i_uncached);
W
Wei Wang 已提交
154
		atomic_dec(&net->ipv6.rt6_stats->fib_rt_uncache);
155 156 157 158 159 160 161 162 163
		spin_unlock_bh(&ul->lock);
	}
}

static void rt6_uncached_list_flush_dev(struct net *net, struct net_device *dev)
{
	struct net_device *loopback_dev = net->loopback_dev;
	int cpu;

164 165 166
	if (dev == loopback_dev)
		return;

167 168 169 170 171 172 173 174 175
	for_each_possible_cpu(cpu) {
		struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
		struct rt6_info *rt;

		spin_lock_bh(&ul->lock);
		list_for_each_entry(rt, &ul->head, rt6i_uncached) {
			struct inet6_dev *rt_idev = rt->rt6i_idev;
			struct net_device *rt_dev = rt->dst.dev;

176
			if (rt_idev->dev == dev) {
177 178 179 180
				rt->rt6i_idev = in6_dev_get(loopback_dev);
				in6_dev_put(rt_idev);
			}

181
			if (rt_dev == dev) {
182 183 184 185 186 187 188 189 190
				rt->dst.dev = loopback_dev;
				dev_hold(rt->dst.dev);
				dev_put(rt_dev);
			}
		}
		spin_unlock_bh(&ul->lock);
	}
}

191
static inline const void *choose_neigh_daddr(const struct in6_addr *p,
192 193
					     struct sk_buff *skb,
					     const void *daddr)
194
{
D
David S. Miller 已提交
195
	if (!ipv6_addr_any(p))
196
		return (const void *) p;
197 198
	else if (skb)
		return &ipv6_hdr(skb)->daddr;
199 200 201
	return daddr;
}

202 203 204 205
struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw,
				   struct net_device *dev,
				   struct sk_buff *skb,
				   const void *daddr)
206
{
207 208
	struct neighbour *n;

209 210
	daddr = choose_neigh_daddr(gw, skb, daddr);
	n = __ipv6_neigh_lookup(dev, daddr);
211 212
	if (n)
		return n;
213 214 215 216 217 218 219 220 221 222
	return neigh_create(&nd_tbl, daddr, dev);
}

static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst,
					      struct sk_buff *skb,
					      const void *daddr)
{
	const struct rt6_info *rt = container_of(dst, struct rt6_info, dst);

	return ip6_neigh_lookup(&rt->rt6i_gateway, dst->dev, skb, daddr);
223 224
}

225 226 227 228 229
static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr)
{
	struct net_device *dev = dst->dev;
	struct rt6_info *rt = (struct rt6_info *)dst;

230
	daddr = choose_neigh_daddr(&rt->rt6i_gateway, NULL, daddr);
231 232 233 234 235 236 237 238 239
	if (!daddr)
		return;
	if (dev->flags & (IFF_NOARP | IFF_LOOPBACK))
		return;
	if (ipv6_addr_is_multicast((const struct in6_addr *)daddr))
		return;
	__ipv6_confirm_neigh(dev, daddr);
}

240
static struct dst_ops ip6_dst_ops_template = {
L
Linus Torvalds 已提交
241 242 243 244
	.family			=	AF_INET6,
	.gc			=	ip6_dst_gc,
	.gc_thresh		=	1024,
	.check			=	ip6_dst_check,
245
	.default_advmss		=	ip6_default_advmss,
246
	.mtu			=	ip6_mtu,
247
	.cow_metrics		=	dst_cow_metrics_generic,
L
Linus Torvalds 已提交
248 249 250 251 252
	.destroy		=	ip6_dst_destroy,
	.ifdown			=	ip6_dst_ifdown,
	.negative_advice	=	ip6_negative_advice,
	.link_failure		=	ip6_link_failure,
	.update_pmtu		=	ip6_rt_update_pmtu,
253
	.redirect		=	rt6_do_redirect,
254
	.local_out		=	__ip6_local_out,
255
	.neigh_lookup		=	ip6_dst_neigh_lookup,
256
	.confirm_neigh		=	ip6_confirm_neigh,
L
Linus Torvalds 已提交
257 258
};

259
static unsigned int ip6_blackhole_mtu(const struct dst_entry *dst)
260
{
261 262 263
	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);

	return mtu ? : dst->dev->mtu;
264 265
}

266 267
static void ip6_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
					 struct sk_buff *skb, u32 mtu)
268 269 270
{
}

271 272
static void ip6_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
				      struct sk_buff *skb)
273 274 275
{
}

276 277 278 279
static struct dst_ops ip6_dst_blackhole_ops = {
	.family			=	AF_INET6,
	.destroy		=	ip6_dst_destroy,
	.check			=	ip6_dst_check,
280
	.mtu			=	ip6_blackhole_mtu,
281
	.default_advmss		=	ip6_default_advmss,
282
	.update_pmtu		=	ip6_rt_blackhole_update_pmtu,
283
	.redirect		=	ip6_rt_blackhole_redirect,
284
	.cow_metrics		=	dst_cow_metrics_generic,
285
	.neigh_lookup		=	ip6_dst_neigh_lookup,
286 287
};

288
static const u32 ip6_template_metrics[RTAX_MAX] = {
L
Li RongQing 已提交
289
	[RTAX_HOPLIMIT - 1] = 0,
290 291
};

292
static const struct fib6_info fib6_null_entry_template = {
293 294 295 296
	.fib6_flags	= (RTF_REJECT | RTF_NONEXTHOP),
	.fib6_protocol  = RTPROT_KERNEL,
	.fib6_metric	= ~(u32)0,
	.fib6_ref	= ATOMIC_INIT(1),
D
David Ahern 已提交
297 298 299 300
	.fib6_type	= RTN_UNREACHABLE,
	.fib6_metrics	= (struct dst_metrics *)&dst_default_metrics,
};

301
static const struct rt6_info ip6_null_entry_template = {
302 303 304
	.dst = {
		.__refcnt	= ATOMIC_INIT(1),
		.__use		= 1,
305
		.obsolete	= DST_OBSOLETE_FORCE_CHK,
306 307 308
		.error		= -ENETUNREACH,
		.input		= ip6_pkt_discard,
		.output		= ip6_pkt_discard_out,
L
Linus Torvalds 已提交
309 310 311 312
	},
	.rt6i_flags	= (RTF_REJECT | RTF_NONEXTHOP),
};

T
Thomas Graf 已提交
313 314
#ifdef CONFIG_IPV6_MULTIPLE_TABLES

315
static const struct rt6_info ip6_prohibit_entry_template = {
316 317 318
	.dst = {
		.__refcnt	= ATOMIC_INIT(1),
		.__use		= 1,
319
		.obsolete	= DST_OBSOLETE_FORCE_CHK,
320 321 322
		.error		= -EACCES,
		.input		= ip6_pkt_prohibit,
		.output		= ip6_pkt_prohibit_out,
T
Thomas Graf 已提交
323 324 325 326
	},
	.rt6i_flags	= (RTF_REJECT | RTF_NONEXTHOP),
};

327
static const struct rt6_info ip6_blk_hole_entry_template = {
328 329 330
	.dst = {
		.__refcnt	= ATOMIC_INIT(1),
		.__use		= 1,
331
		.obsolete	= DST_OBSOLETE_FORCE_CHK,
332 333
		.error		= -EINVAL,
		.input		= dst_discard,
E
Eric W. Biederman 已提交
334
		.output		= dst_discard_out,
T
Thomas Graf 已提交
335 336 337 338 339 340
	},
	.rt6i_flags	= (RTF_REJECT | RTF_NONEXTHOP),
};

#endif

341 342 343 344 345 346 347 348
static void rt6_info_init(struct rt6_info *rt)
{
	struct dst_entry *dst = &rt->dst;

	memset(dst + 1, 0, sizeof(*rt) - sizeof(*dst));
	INIT_LIST_HEAD(&rt->rt6i_uncached);
}

L
Linus Torvalds 已提交
349
/* allocate dst with ip6_dst_ops */
350 351
struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev,
			       int flags)
L
Linus Torvalds 已提交
352
{
353
	struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev,
W
Wei Wang 已提交
354
					1, DST_OBSOLETE_FORCE_CHK, flags);
355

W
Wei Wang 已提交
356
	if (rt) {
357
		rt6_info_init(rt);
W
Wei Wang 已提交
358 359
		atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
	}
360

361
	return rt;
L
Linus Torvalds 已提交
362
}
363
EXPORT_SYMBOL(ip6_dst_alloc);
M
Martin KaFai Lau 已提交
364

L
Linus Torvalds 已提交
365 366 367
static void ip6_dst_destroy(struct dst_entry *dst)
{
	struct rt6_info *rt = (struct rt6_info *)dst;
368
	struct fib6_info *from;
369
	struct inet6_dev *idev;
L
Linus Torvalds 已提交
370

371
	dst_destroy_metrics_generic(dst);
372 373 374
	rt6_uncached_list_del(rt);

	idev = rt->rt6i_idev;
375
	if (idev) {
L
Linus Torvalds 已提交
376 377
		rt->rt6i_idev = NULL;
		in6_dev_put(idev);
378
	}
379

380 381 382
	rcu_read_lock();
	from = rcu_dereference(rt->from);
	rcu_assign_pointer(rt->from, NULL);
383
	fib6_info_release(from);
384
	rcu_read_unlock();
385 386
}

L
Linus Torvalds 已提交
387 388 389 390 391
static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
			   int how)
{
	struct rt6_info *rt = (struct rt6_info *)dst;
	struct inet6_dev *idev = rt->rt6i_idev;
392
	struct net_device *loopback_dev =
393
		dev_net(dev)->loopback_dev;
L
Linus Torvalds 已提交
394

395 396 397 398 399
	if (idev && idev->dev != loopback_dev) {
		struct inet6_dev *loopback_idev = in6_dev_get(loopback_dev);
		if (loopback_idev) {
			rt->rt6i_idev = loopback_idev;
			in6_dev_put(idev);
400
		}
L
Linus Torvalds 已提交
401 402 403
	}
}

404 405 406 407 408 409 410 411
static bool __rt6_check_expired(const struct rt6_info *rt)
{
	if (rt->rt6i_flags & RTF_EXPIRES)
		return time_after(jiffies, rt->dst.expires);
	else
		return false;
}

412
static bool rt6_check_expired(const struct rt6_info *rt)
L
Linus Torvalds 已提交
413
{
414 415 416 417
	struct fib6_info *from;

	from = rcu_dereference(rt->from);

418 419
	if (rt->rt6i_flags & RTF_EXPIRES) {
		if (time_after(jiffies, rt->dst.expires))
420
			return true;
421
	} else if (from) {
422
		return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK ||
423
			fib6_check_expired(from);
424
	}
425
	return false;
L
Linus Torvalds 已提交
426 427
}

428 429 430 431 432
struct fib6_info *fib6_multipath_select(const struct net *net,
					struct fib6_info *match,
					struct flowi6 *fl6, int oif,
					const struct sk_buff *skb,
					int strict)
433
{
434
	struct fib6_info *sibling, *next_sibling;
435

436 437 438 439
	/* We might have already computed the hash for ICMPv6 errors. In such
	 * case it will always be non-zero. Otherwise now is the time to do it.
	 */
	if (!fl6->mp_hash)
440
		fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL);
441

442
	if (fl6->mp_hash <= atomic_read(&match->fib6_nh.nh_upper_bound))
443 444
		return match;

445 446
	list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings,
				 fib6_siblings) {
447 448 449 450
		int nh_upper_bound;

		nh_upper_bound = atomic_read(&sibling->fib6_nh.nh_upper_bound);
		if (fl6->mp_hash > nh_upper_bound)
451 452 453 454 455 456 457
			continue;
		if (rt6_score_route(sibling, oif, strict) < 0)
			break;
		match = sibling;
		break;
	}

458 459 460
	return match;
}

L
Linus Torvalds 已提交
461
/*
462
 *	Route lookup. rcu_read_lock() should be held.
L
Linus Torvalds 已提交
463 464
 */

465 466
static inline struct fib6_info *rt6_device_match(struct net *net,
						 struct fib6_info *rt,
467
						    const struct in6_addr *saddr,
L
Linus Torvalds 已提交
468
						    int oif,
469
						    int flags)
L
Linus Torvalds 已提交
470
{
471
	struct fib6_info *sprt;
L
Linus Torvalds 已提交
472

473 474
	if (!oif && ipv6_addr_any(saddr) &&
	    !(rt->fib6_nh.nh_flags & RTNH_F_DEAD))
475
		return rt;
476

477
	for (sprt = rt; sprt; sprt = rcu_dereference(sprt->fib6_next)) {
478
		const struct net_device *dev = sprt->fib6_nh.nh_dev;
479

480
		if (sprt->fib6_nh.nh_flags & RTNH_F_DEAD)
481 482
			continue;

483
		if (oif) {
L
Linus Torvalds 已提交
484 485
			if (dev->ifindex == oif)
				return sprt;
486 487 488 489
		} else {
			if (ipv6_chk_addr(net, saddr, dev,
					  flags & RT6_LOOKUP_F_IFACE))
				return sprt;
L
Linus Torvalds 已提交
490
		}
491
	}
L
Linus Torvalds 已提交
492

493 494
	if (oif && flags & RT6_LOOKUP_F_IFACE)
		return net->ipv6.fib6_null_entry;
495

D
David Ahern 已提交
496
	return rt->fib6_nh.nh_flags & RTNH_F_DEAD ? net->ipv6.fib6_null_entry : rt;
L
Linus Torvalds 已提交
497 498
}

499
#ifdef CONFIG_IPV6_ROUTER_PREF
500 501 502 503 504 505 506 507 508 509 510 511 512
struct __rt6_probe_work {
	struct work_struct work;
	struct in6_addr target;
	struct net_device *dev;
};

static void rt6_probe_deferred(struct work_struct *w)
{
	struct in6_addr mcaddr;
	struct __rt6_probe_work *work =
		container_of(w, struct __rt6_probe_work, work);

	addrconf_addr_solict_mult(&work->target, &mcaddr);
513
	ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0);
514
	dev_put(work->dev);
515
	kfree(work);
516 517
}

518
static void rt6_probe(struct fib6_info *rt)
519
{
520
	struct __rt6_probe_work *work;
521
	const struct in6_addr *nh_gw;
522
	struct neighbour *neigh;
523 524
	struct net_device *dev;

525 526 527 528 529 530 531 532
	/*
	 * Okay, this does not seem to be appropriate
	 * for now, however, we need to check if it
	 * is really so; aka Router Reachability Probing.
	 *
	 * Router Reachability Probe MUST be rate-limited
	 * to no more than one per minute.
	 */
533
	if (!rt || !(rt->fib6_flags & RTF_GATEWAY))
534
		return;
535 536 537

	nh_gw = &rt->fib6_nh.nh_gw;
	dev = rt->fib6_nh.nh_dev;
538
	rcu_read_lock_bh();
539
	neigh = __ipv6_neigh_lookup_noref(dev, nh_gw);
540
	if (neigh) {
D
David Ahern 已提交
541 542
		struct inet6_dev *idev;

543 544 545
		if (neigh->nud_state & NUD_VALID)
			goto out;

D
David Ahern 已提交
546
		idev = __in6_dev_get(dev);
547
		work = NULL;
548
		write_lock(&neigh->lock);
549 550
		if (!(neigh->nud_state & NUD_VALID) &&
		    time_after(jiffies,
D
David Ahern 已提交
551
			       neigh->updated + idev->cnf.rtr_probe_interval)) {
552 553 554
			work = kmalloc(sizeof(*work), GFP_ATOMIC);
			if (work)
				__neigh_set_probe_once(neigh);
555
		}
556
		write_unlock(&neigh->lock);
557 558
	} else {
		work = kmalloc(sizeof(*work), GFP_ATOMIC);
559
	}
560 561 562

	if (work) {
		INIT_WORK(&work->work, rt6_probe_deferred);
563 564 565
		work->target = *nh_gw;
		dev_hold(dev);
		work->dev = dev;
566 567 568
		schedule_work(&work->work);
	}

569
out:
570
	rcu_read_unlock_bh();
571 572
}
#else
573
static inline void rt6_probe(struct fib6_info *rt)
574 575 576 577
{
}
#endif

L
Linus Torvalds 已提交
578
/*
579
 * Default Router Selection (RFC 2461 6.3.6)
L
Linus Torvalds 已提交
580
 */
581
static inline int rt6_check_dev(struct fib6_info *rt, int oif)
582
{
583 584
	const struct net_device *dev = rt->fib6_nh.nh_dev;

585
	if (!oif || dev->ifindex == oif)
586
		return 2;
587
	return 0;
588
}
L
Linus Torvalds 已提交
589

590
static inline enum rt6_nud_state rt6_check_neigh(struct fib6_info *rt)
L
Linus Torvalds 已提交
591
{
592
	enum rt6_nud_state ret = RT6_NUD_FAIL_HARD;
593
	struct neighbour *neigh;
594

595 596
	if (rt->fib6_flags & RTF_NONEXTHOP ||
	    !(rt->fib6_flags & RTF_GATEWAY))
597
		return RT6_NUD_SUCCEED;
598 599

	rcu_read_lock_bh();
600 601
	neigh = __ipv6_neigh_lookup_noref(rt->fib6_nh.nh_dev,
					  &rt->fib6_nh.nh_gw);
602 603
	if (neigh) {
		read_lock(&neigh->lock);
604
		if (neigh->nud_state & NUD_VALID)
605
			ret = RT6_NUD_SUCCEED;
606
#ifdef CONFIG_IPV6_ROUTER_PREF
607
		else if (!(neigh->nud_state & NUD_FAILED))
608
			ret = RT6_NUD_SUCCEED;
J
Jiri Benc 已提交
609 610
		else
			ret = RT6_NUD_FAIL_PROBE;
611
#endif
612
		read_unlock(&neigh->lock);
613 614
	} else {
		ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ?
J
Jiri Benc 已提交
615
		      RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR;
616
	}
617 618
	rcu_read_unlock_bh();

619
	return ret;
L
Linus Torvalds 已提交
620 621
}

622
static int rt6_score_route(struct fib6_info *rt, int oif, int strict)
L
Linus Torvalds 已提交
623
{
624
	int m;
625

626
	m = rt6_check_dev(rt, oif);
627
	if (!m && (strict & RT6_LOOKUP_F_IFACE))
628
		return RT6_NUD_FAIL_HARD;
629
#ifdef CONFIG_IPV6_ROUTER_PREF
630
	m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(rt->fib6_flags)) << 2;
631
#endif
632 633 634 635 636
	if (strict & RT6_LOOKUP_F_REACHABLE) {
		int n = rt6_check_neigh(rt);
		if (n < 0)
			return n;
	}
637 638 639
	return m;
}

D
David Ahern 已提交
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
/* called with rc_read_lock held */
static inline bool fib6_ignore_linkdown(const struct fib6_info *f6i)
{
	const struct net_device *dev = fib6_info_nh_dev(f6i);
	bool rc = false;

	if (dev) {
		const struct inet6_dev *idev = __in6_dev_get(dev);

		rc = !!idev->cnf.ignore_routes_with_linkdown;
	}

	return rc;
}

655 656
static struct fib6_info *find_match(struct fib6_info *rt, int oif, int strict,
				   int *mpri, struct fib6_info *match,
657
				   bool *do_rr)
658
{
659
	int m;
660
	bool match_do_rr = false;
661

662
	if (rt->fib6_nh.nh_flags & RTNH_F_DEAD)
663 664
		goto out;

D
David Ahern 已提交
665
	if (fib6_ignore_linkdown(rt) &&
666
	    rt->fib6_nh.nh_flags & RTNH_F_LINKDOWN &&
667
	    !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE))
668
		goto out;
669

670
	if (fib6_check_expired(rt))
671 672 673
		goto out;

	m = rt6_score_route(rt, oif, strict);
J
Jiri Benc 已提交
674
	if (m == RT6_NUD_FAIL_DO_RR) {
675 676
		match_do_rr = true;
		m = 0; /* lowest valid score */
J
Jiri Benc 已提交
677
	} else if (m == RT6_NUD_FAIL_HARD) {
678
		goto out;
679 680 681 682
	}

	if (strict & RT6_LOOKUP_F_REACHABLE)
		rt6_probe(rt);
683

J
Jiri Benc 已提交
684
	/* note that m can be RT6_NUD_FAIL_PROBE at this point */
685
	if (m > *mpri) {
686
		*do_rr = match_do_rr;
687 688 689 690 691 692 693
		*mpri = m;
		match = rt;
	}
out:
	return match;
}

694 695 696
static struct fib6_info *find_rr_leaf(struct fib6_node *fn,
				     struct fib6_info *leaf,
				     struct fib6_info *rr_head,
697 698
				     u32 metric, int oif, int strict,
				     bool *do_rr)
699
{
700
	struct fib6_info *rt, *match, *cont;
701
	int mpri = -1;
L
Linus Torvalds 已提交
702

703
	match = NULL;
704
	cont = NULL;
705
	for (rt = rr_head; rt; rt = rcu_dereference(rt->fib6_next)) {
706
		if (rt->fib6_metric != metric) {
707 708 709 710 711 712 713
			cont = rt;
			break;
		}

		match = find_match(rt, oif, strict, &mpri, match, do_rr);
	}

714
	for (rt = leaf; rt && rt != rr_head;
715
	     rt = rcu_dereference(rt->fib6_next)) {
716
		if (rt->fib6_metric != metric) {
717 718 719 720
			cont = rt;
			break;
		}

721
		match = find_match(rt, oif, strict, &mpri, match, do_rr);
722 723 724 725 726
	}

	if (match || !cont)
		return match;

727
	for (rt = cont; rt; rt = rcu_dereference(rt->fib6_next))
728
		match = find_match(rt, oif, strict, &mpri, match, do_rr);
L
Linus Torvalds 已提交
729

730 731
	return match;
}
L
Linus Torvalds 已提交
732

733
static struct fib6_info *rt6_select(struct net *net, struct fib6_node *fn,
W
Wei Wang 已提交
734
				   int oif, int strict)
735
{
736 737
	struct fib6_info *leaf = rcu_dereference(fn->leaf);
	struct fib6_info *match, *rt0;
738
	bool do_rr = false;
739
	int key_plen;
L
Linus Torvalds 已提交
740

D
David Ahern 已提交
741 742
	if (!leaf || leaf == net->ipv6.fib6_null_entry)
		return net->ipv6.fib6_null_entry;
W
Wei Wang 已提交
743

744
	rt0 = rcu_dereference(fn->rr_ptr);
745
	if (!rt0)
746
		rt0 = leaf;
L
Linus Torvalds 已提交
747

748 749 750 751 752
	/* Double check to make sure fn is not an intermediate node
	 * and fn->leaf does not points to its child's leaf
	 * (This might happen if all routes under fn are deleted from
	 * the tree and fib6_repair_tree() is called on the node.)
	 */
753
	key_plen = rt0->fib6_dst.plen;
754
#ifdef CONFIG_IPV6_SUBTREES
755 756
	if (rt0->fib6_src.plen)
		key_plen = rt0->fib6_src.plen;
757 758
#endif
	if (fn->fn_bit != key_plen)
D
David Ahern 已提交
759
		return net->ipv6.fib6_null_entry;
760

761
	match = find_rr_leaf(fn, leaf, rt0, rt0->fib6_metric, oif, strict,
762
			     &do_rr);
L
Linus Torvalds 已提交
763

764
	if (do_rr) {
765
		struct fib6_info *next = rcu_dereference(rt0->fib6_next);
766

767
		/* no entries matched; do round-robin */
768
		if (!next || next->fib6_metric != rt0->fib6_metric)
W
Wei Wang 已提交
769
			next = leaf;
770

771
		if (next != rt0) {
772
			spin_lock_bh(&leaf->fib6_table->tb6_lock);
773
			/* make sure next is not being deleted from the tree */
774
			if (next->fib6_node)
775
				rcu_assign_pointer(fn->rr_ptr, next);
776
			spin_unlock_bh(&leaf->fib6_table->tb6_lock);
777
		}
L
Linus Torvalds 已提交
778 779
	}

D
David Ahern 已提交
780
	return match ? match : net->ipv6.fib6_null_entry;
L
Linus Torvalds 已提交
781 782
}

783
static bool rt6_is_gw_or_nonexthop(const struct fib6_info *rt)
784
{
785
	return (rt->fib6_flags & (RTF_NONEXTHOP | RTF_GATEWAY));
786 787
}

788 789
#ifdef CONFIG_IPV6_ROUTE_INFO
int rt6_route_rcv(struct net_device *dev, u8 *opt, int len,
790
		  const struct in6_addr *gwaddr)
791
{
792
	struct net *net = dev_net(dev);
793 794 795
	struct route_info *rinfo = (struct route_info *) opt;
	struct in6_addr prefix_buf, *prefix;
	unsigned int pref;
796
	unsigned long lifetime;
797
	struct fib6_info *rt;
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819

	if (len < sizeof(struct route_info)) {
		return -EINVAL;
	}

	/* Sanity check for prefix_len and length */
	if (rinfo->length > 3) {
		return -EINVAL;
	} else if (rinfo->prefix_len > 128) {
		return -EINVAL;
	} else if (rinfo->prefix_len > 64) {
		if (rinfo->length < 2) {
			return -EINVAL;
		}
	} else if (rinfo->prefix_len > 0) {
		if (rinfo->length < 1) {
			return -EINVAL;
		}
	}

	pref = rinfo->route_pref;
	if (pref == ICMPV6_ROUTER_PREF_INVALID)
820
		return -EINVAL;
821

822
	lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ);
823 824 825 826 827 828 829 830 831 832 833

	if (rinfo->length == 3)
		prefix = (struct in6_addr *)rinfo->prefix;
	else {
		/* this function is safe */
		ipv6_addr_prefix(&prefix_buf,
				 (struct in6_addr *)rinfo->prefix,
				 rinfo->prefix_len);
		prefix = &prefix_buf;
	}

834
	if (rinfo->prefix_len == 0)
835
		rt = rt6_get_dflt_router(net, gwaddr, dev);
836 837
	else
		rt = rt6_get_route_info(net, prefix, rinfo->prefix_len,
838
					gwaddr, dev);
839 840

	if (rt && !lifetime) {
841
		ip6_del_rt(net, rt);
842 843 844 845
		rt = NULL;
	}

	if (!rt && lifetime)
846 847
		rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr,
					dev, pref);
848
	else if (rt)
849 850
		rt->fib6_flags = RTF_ROUTEINFO |
				 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref);
851 852

	if (rt) {
853
		if (!addrconf_finite_timeout(lifetime))
854
			fib6_clean_expires(rt);
855
		else
856
			fib6_set_expires(rt, jiffies + HZ * lifetime);
857

858
		fib6_info_release(rt);
859 860 861 862 863
	}
	return 0;
}
#endif

864 865 866 867 868
/*
 *	Misc support functions
 */

/* called with rcu_lock held */
869
static struct net_device *ip6_rt_get_dev_rcu(struct fib6_info *rt)
870
{
871
	struct net_device *dev = rt->fib6_nh.nh_dev;
872

873
	if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) {
874 875 876 877 878
		/* for copies of local routes, dst->dev needs to be the
		 * device if it is a master device, the master device if
		 * device is enslaved, and the loopback as the default
		 */
		if (netif_is_l3_slave(dev) &&
879
		    !rt6_need_strict(&rt->fib6_dst.addr))
880 881 882 883 884 885 886 887 888 889 890
			dev = l3mdev_master_dev_rcu(dev);
		else if (!netif_is_l3_master(dev))
			dev = dev_net(dev)->loopback_dev;
		/* last case is netif_is_l3_master(dev) is true in which
		 * case we want dev returned to be dev
		 */
	}

	return dev;
}

891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
static const int fib6_prop[RTN_MAX + 1] = {
	[RTN_UNSPEC]	= 0,
	[RTN_UNICAST]	= 0,
	[RTN_LOCAL]	= 0,
	[RTN_BROADCAST]	= 0,
	[RTN_ANYCAST]	= 0,
	[RTN_MULTICAST]	= 0,
	[RTN_BLACKHOLE]	= -EINVAL,
	[RTN_UNREACHABLE] = -EHOSTUNREACH,
	[RTN_PROHIBIT]	= -EACCES,
	[RTN_THROW]	= -EAGAIN,
	[RTN_NAT]	= -EINVAL,
	[RTN_XRESOLVE]	= -EINVAL,
};

static int ip6_rt_type_to_error(u8 fib6_type)
{
	return fib6_prop[fib6_type];
}

911
static unsigned short fib6_info_dst_flags(struct fib6_info *rt)
912 913 914 915 916 917 918 919 920 921 922 923 924
{
	unsigned short flags = 0;

	if (rt->dst_nocount)
		flags |= DST_NOCOUNT;
	if (rt->dst_nopolicy)
		flags |= DST_NOPOLICY;
	if (rt->dst_host)
		flags |= DST_HOST;

	return flags;
}

925
static void ip6_rt_init_dst_reject(struct rt6_info *rt, struct fib6_info *ort)
926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946
{
	rt->dst.error = ip6_rt_type_to_error(ort->fib6_type);

	switch (ort->fib6_type) {
	case RTN_BLACKHOLE:
		rt->dst.output = dst_discard_out;
		rt->dst.input = dst_discard;
		break;
	case RTN_PROHIBIT:
		rt->dst.output = ip6_pkt_prohibit_out;
		rt->dst.input = ip6_pkt_prohibit;
		break;
	case RTN_THROW:
	case RTN_UNREACHABLE:
	default:
		rt->dst.output = ip6_pkt_discard_out;
		rt->dst.input = ip6_pkt_discard;
		break;
	}
}

947
static void ip6_rt_init_dst(struct rt6_info *rt, struct fib6_info *ort)
948
{
949 950
	rt->dst.flags |= fib6_info_dst_flags(ort);

951
	if (ort->fib6_flags & RTF_REJECT) {
952 953 954 955 956 957 958 959 960
		ip6_rt_init_dst_reject(rt, ort);
		return;
	}

	rt->dst.error = 0;
	rt->dst.output = ip6_output;

	if (ort->fib6_type == RTN_LOCAL) {
		rt->dst.input = ip6_input;
961
	} else if (ipv6_addr_type(&ort->fib6_dst.addr) & IPV6_ADDR_MULTICAST) {
962 963 964 965 966 967 968 969 970 971 972 973 974
		rt->dst.input = ip6_mc_input;
	} else {
		rt->dst.input = ip6_forward;
	}

	if (ort->fib6_nh.nh_lwtstate) {
		rt->dst.lwtstate = lwtstate_get(ort->fib6_nh.nh_lwtstate);
		lwtunnel_set_redirect(&rt->dst);
	}

	rt->dst.lastuse = jiffies;
}

975
static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from)
976 977
{
	rt->rt6i_flags &= ~RTF_EXPIRES;
978
	fib6_info_hold(from);
979
	rcu_assign_pointer(rt->from, from);
980 981 982 983 984
	dst_init_metrics(&rt->dst, from->fib6_metrics->metrics, true);
	if (from->fib6_metrics != &dst_default_metrics) {
		rt->dst._metrics |= DST_METRICS_REFCOUNTED;
		refcount_inc(&from->fib6_metrics->refcnt);
	}
985 986
}

987
static void ip6_rt_copy_init(struct rt6_info *rt, struct fib6_info *ort)
988
{
D
David Ahern 已提交
989 990
	struct net_device *dev = fib6_info_nh_dev(ort);

991 992
	ip6_rt_init_dst(rt, ort);

993
	rt->rt6i_dst = ort->fib6_dst;
D
David Ahern 已提交
994
	rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL;
995
	rt->rt6i_gateway = ort->fib6_nh.nh_gw;
996
	rt->rt6i_flags = ort->fib6_flags;
997 998
	rt6_set_from(rt, ort);
#ifdef CONFIG_IPV6_SUBTREES
999
	rt->rt6i_src = ort->fib6_src;
1000
#endif
1001
	rt->rt6i_prefsrc = ort->fib6_prefsrc;
1002
	rt->dst.lwtstate = lwtstate_get(ort->fib6_nh.nh_lwtstate);
1003 1004
}

M
Martin KaFai Lau 已提交
1005 1006 1007
static struct fib6_node* fib6_backtrack(struct fib6_node *fn,
					struct in6_addr *saddr)
{
1008
	struct fib6_node *pn, *sn;
M
Martin KaFai Lau 已提交
1009 1010 1011
	while (1) {
		if (fn->fn_flags & RTN_TL_ROOT)
			return NULL;
1012 1013 1014
		pn = rcu_dereference(fn->parent);
		sn = FIB6_SUBTREE(pn);
		if (sn && sn != fn)
1015
			fn = fib6_node_lookup(sn, NULL, saddr);
M
Martin KaFai Lau 已提交
1016 1017 1018 1019 1020 1021
		else
			fn = pn;
		if (fn->fn_flags & RTN_RTINFO)
			return fn;
	}
}
T
Thomas Graf 已提交
1022

1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
static bool ip6_hold_safe(struct net *net, struct rt6_info **prt,
			  bool null_fallback)
{
	struct rt6_info *rt = *prt;

	if (dst_hold_safe(&rt->dst))
		return true;
	if (null_fallback) {
		rt = net->ipv6.ip6_null_entry;
		dst_hold(&rt->dst);
	} else {
		rt = NULL;
	}
	*prt = rt;
	return false;
}

1040
/* called with rcu_lock held */
1041
static struct rt6_info *ip6_create_rt_rcu(struct fib6_info *rt)
1042
{
1043
	unsigned short flags = fib6_info_dst_flags(rt);
1044 1045 1046
	struct net_device *dev = rt->fib6_nh.nh_dev;
	struct rt6_info *nrt;

1047
	nrt = ip6_dst_alloc(dev_net(dev), dev, flags);
1048 1049 1050 1051 1052 1053
	if (nrt)
		ip6_rt_copy_init(nrt, rt);

	return nrt;
}

1054 1055
static struct rt6_info *ip6_pol_route_lookup(struct net *net,
					     struct fib6_table *table,
D
David Ahern 已提交
1056 1057 1058
					     struct flowi6 *fl6,
					     const struct sk_buff *skb,
					     int flags)
L
Linus Torvalds 已提交
1059
{
1060
	struct fib6_info *f6i;
L
Linus Torvalds 已提交
1061
	struct fib6_node *fn;
1062
	struct rt6_info *rt;
L
Linus Torvalds 已提交
1063

1064 1065 1066
	if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF)
		flags &= ~RT6_LOOKUP_F_IFACE;

1067
	rcu_read_lock();
1068
	fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
T
Thomas Graf 已提交
1069
restart:
1070 1071 1072
	f6i = rcu_dereference(fn->leaf);
	if (!f6i) {
		f6i = net->ipv6.fib6_null_entry;
1073
	} else {
1074
		f6i = rt6_device_match(net, f6i, &fl6->saddr,
1075
				      fl6->flowi6_oif, flags);
1076
		if (f6i->fib6_nsiblings && fl6->flowi6_oif == 0)
1077 1078 1079
			f6i = fib6_multipath_select(net, f6i, fl6,
						    fl6->flowi6_oif, skb,
						    flags);
1080
	}
1081
	if (f6i == net->ipv6.fib6_null_entry) {
M
Martin KaFai Lau 已提交
1082 1083 1084 1085
		fn = fib6_backtrack(fn, &fl6->saddr);
		if (fn)
			goto restart;
	}
1086

1087 1088
	trace_fib6_table_lookup(net, f6i, table, fl6);

1089
	/* Search through exception table */
1090 1091
	rt = rt6_find_cached_rt(f6i, &fl6->daddr, &fl6->saddr);
	if (rt) {
1092 1093
		if (ip6_hold_safe(net, &rt, true))
			dst_use_noref(&rt->dst, jiffies);
1094
	} else if (f6i == net->ipv6.fib6_null_entry) {
1095 1096
		rt = net->ipv6.ip6_null_entry;
		dst_hold(&rt->dst);
1097 1098 1099 1100 1101 1102
	} else {
		rt = ip6_create_rt_rcu(f6i);
		if (!rt) {
			rt = net->ipv6.ip6_null_entry;
			dst_hold(&rt->dst);
		}
1103
	}
1104

1105
	rcu_read_unlock();
D
David Ahern 已提交
1106

T
Thomas Graf 已提交
1107 1108 1109
	return rt;
}

1110
struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6,
D
David Ahern 已提交
1111
				   const struct sk_buff *skb, int flags)
F
Florian Westphal 已提交
1112
{
D
David Ahern 已提交
1113
	return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup);
F
Florian Westphal 已提交
1114 1115 1116
}
EXPORT_SYMBOL_GPL(ip6_route_lookup);

1117
struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr,
D
David Ahern 已提交
1118 1119
			    const struct in6_addr *saddr, int oif,
			    const struct sk_buff *skb, int strict)
T
Thomas Graf 已提交
1120
{
1121 1122 1123
	struct flowi6 fl6 = {
		.flowi6_oif = oif,
		.daddr = *daddr,
T
Thomas Graf 已提交
1124 1125
	};
	struct dst_entry *dst;
1126
	int flags = strict ? RT6_LOOKUP_F_IFACE : 0;
T
Thomas Graf 已提交
1127

1128
	if (saddr) {
1129
		memcpy(&fl6.saddr, saddr, sizeof(*saddr));
1130 1131 1132
		flags |= RT6_LOOKUP_F_HAS_SADDR;
	}

D
David Ahern 已提交
1133
	dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup);
T
Thomas Graf 已提交
1134 1135 1136 1137 1138
	if (dst->error == 0)
		return (struct rt6_info *) dst;

	dst_release(dst);

L
Linus Torvalds 已提交
1139 1140
	return NULL;
}
1141 1142
EXPORT_SYMBOL(rt6_lookup);

T
Thomas Graf 已提交
1143
/* ip6_ins_rt is called with FREE table->tb6_lock.
1144 1145 1146
 * It takes new route entry, the addition fails by any reason the
 * route is released.
 * Caller must hold dst before calling it.
L
Linus Torvalds 已提交
1147 1148
 */

1149
static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info,
1150
			struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
1151 1152
{
	int err;
T
Thomas Graf 已提交
1153
	struct fib6_table *table;
L
Linus Torvalds 已提交
1154

1155
	table = rt->fib6_table;
1156
	spin_lock_bh(&table->tb6_lock);
1157
	err = fib6_add(&table->tb6_root, rt, info, extack);
1158
	spin_unlock_bh(&table->tb6_lock);
L
Linus Torvalds 已提交
1159 1160 1161 1162

	return err;
}

1163
int ip6_ins_rt(struct net *net, struct fib6_info *rt)
1164
{
1165
	struct nl_info info = {	.nl_net = net, };
1166

1167
	return __ip6_ins_rt(rt, &info, NULL);
1168 1169
}

1170
static struct rt6_info *ip6_rt_cache_alloc(struct fib6_info *ort,
1171 1172
					   const struct in6_addr *daddr,
					   const struct in6_addr *saddr)
L
Linus Torvalds 已提交
1173
{
1174
	struct net_device *dev;
L
Linus Torvalds 已提交
1175 1176 1177 1178 1179 1180
	struct rt6_info *rt;

	/*
	 *	Clone the route.
	 */

1181
	dev = ip6_rt_get_dev_rcu(ort);
1182
	rt = ip6_dst_alloc(dev_net(dev), dev, 0);
M
Martin KaFai Lau 已提交
1183 1184 1185 1186 1187 1188 1189 1190
	if (!rt)
		return NULL;

	ip6_rt_copy_init(rt, ort);
	rt->rt6i_flags |= RTF_CACHE;
	rt->dst.flags |= DST_HOST;
	rt->rt6i_dst.addr = *daddr;
	rt->rt6i_dst.plen = 128;
L
Linus Torvalds 已提交
1191

M
Martin KaFai Lau 已提交
1192
	if (!rt6_is_gw_or_nonexthop(ort)) {
1193 1194
		if (ort->fib6_dst.plen != 128 &&
		    ipv6_addr_equal(&ort->fib6_dst.addr, daddr))
M
Martin KaFai Lau 已提交
1195
			rt->rt6i_flags |= RTF_ANYCAST;
L
Linus Torvalds 已提交
1196
#ifdef CONFIG_IPV6_SUBTREES
M
Martin KaFai Lau 已提交
1197 1198 1199
		if (rt->rt6i_src.plen && saddr) {
			rt->rt6i_src.addr = *saddr;
			rt->rt6i_src.plen = 128;
1200
		}
M
Martin KaFai Lau 已提交
1201
#endif
1202
	}
L
Linus Torvalds 已提交
1203

1204 1205
	return rt;
}
L
Linus Torvalds 已提交
1206

1207
static struct rt6_info *ip6_rt_pcpu_alloc(struct fib6_info *rt)
M
Martin KaFai Lau 已提交
1208
{
1209
	unsigned short flags = fib6_info_dst_flags(rt);
1210
	struct net_device *dev;
M
Martin KaFai Lau 已提交
1211 1212
	struct rt6_info *pcpu_rt;

1213 1214
	rcu_read_lock();
	dev = ip6_rt_get_dev_rcu(rt);
1215
	pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags);
1216
	rcu_read_unlock();
M
Martin KaFai Lau 已提交
1217 1218 1219 1220 1221 1222 1223
	if (!pcpu_rt)
		return NULL;
	ip6_rt_copy_init(pcpu_rt, rt);
	pcpu_rt->rt6i_flags |= RTF_PCPU;
	return pcpu_rt;
}

1224
/* It should be called with rcu_read_lock() acquired */
1225
static struct rt6_info *rt6_get_pcpu_route(struct fib6_info *rt)
M
Martin KaFai Lau 已提交
1226
{
1227
	struct rt6_info *pcpu_rt, **p;
M
Martin KaFai Lau 已提交
1228 1229 1230 1231

	p = this_cpu_ptr(rt->rt6i_pcpu);
	pcpu_rt = *p;

1232 1233
	if (pcpu_rt)
		ip6_hold_safe(NULL, &pcpu_rt, false);
1234

1235 1236 1237
	return pcpu_rt;
}

1238
static struct rt6_info *rt6_make_pcpu_route(struct net *net,
1239
					    struct fib6_info *rt)
1240 1241
{
	struct rt6_info *pcpu_rt, *prev, **p;
M
Martin KaFai Lau 已提交
1242 1243 1244

	pcpu_rt = ip6_rt_pcpu_alloc(rt);
	if (!pcpu_rt) {
1245 1246
		dst_hold(&net->ipv6.ip6_null_entry->dst);
		return net->ipv6.ip6_null_entry;
M
Martin KaFai Lau 已提交
1247 1248
	}

1249 1250 1251
	dst_hold(&pcpu_rt->dst);
	p = this_cpu_ptr(rt->rt6i_pcpu);
	prev = cmpxchg(p, NULL, pcpu_rt);
1252
	BUG_ON(prev);
1253

M
Martin KaFai Lau 已提交
1254 1255 1256
	return pcpu_rt;
}

1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
/* exception hash table implementation
 */
static DEFINE_SPINLOCK(rt6_exception_lock);

/* Remove rt6_ex from hash table and free the memory
 * Caller must hold rt6_exception_lock
 */
static void rt6_remove_exception(struct rt6_exception_bucket *bucket,
				 struct rt6_exception *rt6_ex)
{
1267
	struct net *net;
W
Wei Wang 已提交
1268

1269 1270
	if (!bucket || !rt6_ex)
		return;
1271 1272

	net = dev_net(rt6_ex->rt6i->dst.dev);
1273
	hlist_del_rcu(&rt6_ex->hlist);
1274
	dst_release(&rt6_ex->rt6i->dst);
1275 1276 1277
	kfree_rcu(rt6_ex, rcu);
	WARN_ON_ONCE(!bucket->depth);
	bucket->depth--;
W
Wei Wang 已提交
1278
	net->ipv6.rt6_stats->fib_rt_cache--;
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381
}

/* Remove oldest rt6_ex in bucket and free the memory
 * Caller must hold rt6_exception_lock
 */
static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)
{
	struct rt6_exception *rt6_ex, *oldest = NULL;

	if (!bucket)
		return;

	hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
		if (!oldest || time_before(rt6_ex->stamp, oldest->stamp))
			oldest = rt6_ex;
	}
	rt6_remove_exception(bucket, oldest);
}

static u32 rt6_exception_hash(const struct in6_addr *dst,
			      const struct in6_addr *src)
{
	static u32 seed __read_mostly;
	u32 val;

	net_get_random_once(&seed, sizeof(seed));
	val = jhash(dst, sizeof(*dst), seed);

#ifdef CONFIG_IPV6_SUBTREES
	if (src)
		val = jhash(src, sizeof(*src), val);
#endif
	return hash_32(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT);
}

/* Helper function to find the cached rt in the hash table
 * and update bucket pointer to point to the bucket for this
 * (daddr, saddr) pair
 * Caller must hold rt6_exception_lock
 */
static struct rt6_exception *
__rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket,
			      const struct in6_addr *daddr,
			      const struct in6_addr *saddr)
{
	struct rt6_exception *rt6_ex;
	u32 hval;

	if (!(*bucket) || !daddr)
		return NULL;

	hval = rt6_exception_hash(daddr, saddr);
	*bucket += hval;

	hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) {
		struct rt6_info *rt6 = rt6_ex->rt6i;
		bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);

#ifdef CONFIG_IPV6_SUBTREES
		if (matched && saddr)
			matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
#endif
		if (matched)
			return rt6_ex;
	}
	return NULL;
}

/* Helper function to find the cached rt in the hash table
 * and update bucket pointer to point to the bucket for this
 * (daddr, saddr) pair
 * Caller must hold rcu_read_lock()
 */
static struct rt6_exception *
__rt6_find_exception_rcu(struct rt6_exception_bucket **bucket,
			 const struct in6_addr *daddr,
			 const struct in6_addr *saddr)
{
	struct rt6_exception *rt6_ex;
	u32 hval;

	WARN_ON_ONCE(!rcu_read_lock_held());

	if (!(*bucket) || !daddr)
		return NULL;

	hval = rt6_exception_hash(daddr, saddr);
	*bucket += hval;

	hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) {
		struct rt6_info *rt6 = rt6_ex->rt6i;
		bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);

#ifdef CONFIG_IPV6_SUBTREES
		if (matched && saddr)
			matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
#endif
		if (matched)
			return rt6_ex;
	}
	return NULL;
}

1382
static unsigned int fib6_mtu(const struct fib6_info *rt)
1383 1384 1385
{
	unsigned int mtu;

D
David Ahern 已提交
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
	if (rt->fib6_pmtu) {
		mtu = rt->fib6_pmtu;
	} else {
		struct net_device *dev = fib6_info_nh_dev(rt);
		struct inet6_dev *idev;

		rcu_read_lock();
		idev = __in6_dev_get(dev);
		mtu = idev->cnf.mtu6;
		rcu_read_unlock();
	}

1398 1399 1400 1401 1402
	mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);

	return mtu - lwtunnel_headroom(rt->fib6_nh.nh_lwtstate, mtu);
}

1403
static int rt6_insert_exception(struct rt6_info *nrt,
1404
				struct fib6_info *ort)
1405
{
1406
	struct net *net = dev_net(nrt->dst.dev);
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
	struct rt6_exception_bucket *bucket;
	struct in6_addr *src_key = NULL;
	struct rt6_exception *rt6_ex;
	int err = 0;

	spin_lock_bh(&rt6_exception_lock);

	if (ort->exception_bucket_flushed) {
		err = -EINVAL;
		goto out;
	}

	bucket = rcu_dereference_protected(ort->rt6i_exception_bucket,
					lockdep_is_held(&rt6_exception_lock));
	if (!bucket) {
		bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket),
				 GFP_ATOMIC);
		if (!bucket) {
			err = -ENOMEM;
			goto out;
		}
		rcu_assign_pointer(ort->rt6i_exception_bucket, bucket);
	}

#ifdef CONFIG_IPV6_SUBTREES
	/* rt6i_src.plen != 0 indicates ort is in subtree
	 * and exception table is indexed by a hash of
	 * both rt6i_dst and rt6i_src.
	 * Otherwise, the exception table is indexed by
	 * a hash of only rt6i_dst.
	 */
1438
	if (ort->fib6_src.plen)
1439 1440
		src_key = &nrt->rt6i_src.addr;
#endif
1441 1442 1443 1444

	/* Update rt6i_prefsrc as it could be changed
	 * in rt6_remove_prefsrc()
	 */
1445
	nrt->rt6i_prefsrc = ort->fib6_prefsrc;
1446 1447 1448 1449
	/* rt6_mtu_change() might lower mtu on ort.
	 * Only insert this exception route if its mtu
	 * is less than ort's mtu value.
	 */
1450
	if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(ort)) {
1451 1452 1453
		err = -EINVAL;
		goto out;
	}
1454

1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468
	rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr,
					       src_key);
	if (rt6_ex)
		rt6_remove_exception(bucket, rt6_ex);

	rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC);
	if (!rt6_ex) {
		err = -ENOMEM;
		goto out;
	}
	rt6_ex->rt6i = nrt;
	rt6_ex->stamp = jiffies;
	hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain);
	bucket->depth++;
W
Wei Wang 已提交
1469
	net->ipv6.rt6_stats->fib_rt_cache++;
1470 1471 1472 1473 1474 1475 1476 1477

	if (bucket->depth > FIB6_MAX_DEPTH)
		rt6_exception_remove_oldest(bucket);

out:
	spin_unlock_bh(&rt6_exception_lock);

	/* Update fn->fn_sernum to invalidate all cached dst */
1478
	if (!err) {
1479
		spin_lock_bh(&ort->fib6_table->tb6_lock);
1480
		fib6_update_sernum(net, ort);
1481
		spin_unlock_bh(&ort->fib6_table->tb6_lock);
1482 1483
		fib6_force_start_gc(net);
	}
1484 1485 1486 1487

	return err;
}

1488
void rt6_flush_exceptions(struct fib6_info *rt)
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517
{
	struct rt6_exception_bucket *bucket;
	struct rt6_exception *rt6_ex;
	struct hlist_node *tmp;
	int i;

	spin_lock_bh(&rt6_exception_lock);
	/* Prevent rt6_insert_exception() to recreate the bucket list */
	rt->exception_bucket_flushed = 1;

	bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
				    lockdep_is_held(&rt6_exception_lock));
	if (!bucket)
		goto out;

	for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
		hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist)
			rt6_remove_exception(bucket, rt6_ex);
		WARN_ON_ONCE(bucket->depth);
		bucket++;
	}

out:
	spin_unlock_bh(&rt6_exception_lock);
}

/* Find cached rt in the hash table inside passed in rt
 * Caller has to hold rcu_read_lock()
 */
1518
static struct rt6_info *rt6_find_cached_rt(struct fib6_info *rt,
1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
					   struct in6_addr *daddr,
					   struct in6_addr *saddr)
{
	struct rt6_exception_bucket *bucket;
	struct in6_addr *src_key = NULL;
	struct rt6_exception *rt6_ex;
	struct rt6_info *res = NULL;

	bucket = rcu_dereference(rt->rt6i_exception_bucket);

#ifdef CONFIG_IPV6_SUBTREES
	/* rt6i_src.plen != 0 indicates rt is in subtree
	 * and exception table is indexed by a hash of
	 * both rt6i_dst and rt6i_src.
	 * Otherwise, the exception table is indexed by
	 * a hash of only rt6i_dst.
	 */
1536
	if (rt->fib6_src.plen)
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547
		src_key = saddr;
#endif
	rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key);

	if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i))
		res = rt6_ex->rt6i;

	return res;
}

/* Remove the passed in cached rt from the hash table that contains it */
1548
static int rt6_remove_exception_rt(struct rt6_info *rt)
1549 1550 1551 1552
{
	struct rt6_exception_bucket *bucket;
	struct in6_addr *src_key = NULL;
	struct rt6_exception *rt6_ex;
1553
	struct fib6_info *from;
1554 1555
	int err;

1556
	from = rcu_dereference(rt->from);
1557
	if (!from ||
1558
	    !(rt->rt6i_flags & RTF_CACHE))
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
		return -EINVAL;

	if (!rcu_access_pointer(from->rt6i_exception_bucket))
		return -ENOENT;

	spin_lock_bh(&rt6_exception_lock);
	bucket = rcu_dereference_protected(from->rt6i_exception_bucket,
				    lockdep_is_held(&rt6_exception_lock));
#ifdef CONFIG_IPV6_SUBTREES
	/* rt6i_src.plen != 0 indicates 'from' is in subtree
	 * and exception table is indexed by a hash of
	 * both rt6i_dst and rt6i_src.
	 * Otherwise, the exception table is indexed by
	 * a hash of only rt6i_dst.
	 */
1574
	if (from->fib6_src.plen)
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596
		src_key = &rt->rt6i_src.addr;
#endif
	rt6_ex = __rt6_find_exception_spinlock(&bucket,
					       &rt->rt6i_dst.addr,
					       src_key);
	if (rt6_ex) {
		rt6_remove_exception(bucket, rt6_ex);
		err = 0;
	} else {
		err = -ENOENT;
	}

	spin_unlock_bh(&rt6_exception_lock);
	return err;
}

/* Find rt6_ex which contains the passed in rt cache and
 * refresh its stamp
 */
static void rt6_update_exception_stamp_rt(struct rt6_info *rt)
{
	struct rt6_exception_bucket *bucket;
1597
	struct fib6_info *from = rt->from;
1598 1599 1600 1601
	struct in6_addr *src_key = NULL;
	struct rt6_exception *rt6_ex;

	if (!from ||
1602
	    !(rt->rt6i_flags & RTF_CACHE))
1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
		return;

	rcu_read_lock();
	bucket = rcu_dereference(from->rt6i_exception_bucket);

#ifdef CONFIG_IPV6_SUBTREES
	/* rt6i_src.plen != 0 indicates 'from' is in subtree
	 * and exception table is indexed by a hash of
	 * both rt6i_dst and rt6i_src.
	 * Otherwise, the exception table is indexed by
	 * a hash of only rt6i_dst.
	 */
1615
	if (from->fib6_src.plen)
1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626
		src_key = &rt->rt6i_src.addr;
#endif
	rt6_ex = __rt6_find_exception_rcu(&bucket,
					  &rt->rt6i_dst.addr,
					  src_key);
	if (rt6_ex)
		rt6_ex->stamp = jiffies;

	rcu_read_unlock();
}

1627
static void rt6_exceptions_remove_prefsrc(struct fib6_info *rt)
1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
{
	struct rt6_exception_bucket *bucket;
	struct rt6_exception *rt6_ex;
	int i;

	bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
					lockdep_is_held(&rt6_exception_lock));

	if (bucket) {
		for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
			hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
				rt6_ex->rt6i->rt6i_prefsrc.plen = 0;
			}
			bucket++;
		}
	}
}

1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev,
					 struct rt6_info *rt, int mtu)
{
	/* If the new MTU is lower than the route PMTU, this new MTU will be the
	 * lowest MTU in the path: always allow updating the route PMTU to
	 * reflect PMTU decreases.
	 *
	 * If the new MTU is higher, and the route PMTU is equal to the local
	 * MTU, this means the old MTU is the lowest in the path, so allow
	 * updating it: if other nodes now have lower MTUs, PMTU discovery will
	 * handle this.
	 */

	if (dst_mtu(&rt->dst) >= mtu)
		return true;

	if (dst_mtu(&rt->dst) == idev->cnf.mtu6)
		return true;

	return false;
}

static void rt6_exceptions_update_pmtu(struct inet6_dev *idev,
1669
				       struct fib6_info *rt, int mtu)
1670 1671 1672 1673 1674 1675 1676 1677
{
	struct rt6_exception_bucket *bucket;
	struct rt6_exception *rt6_ex;
	int i;

	bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
					lockdep_is_held(&rt6_exception_lock));

1678 1679 1680 1681 1682 1683 1684 1685
	if (!bucket)
		return;

	for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
		hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
			struct rt6_info *entry = rt6_ex->rt6i;

			/* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected
1686
			 * route), the metrics of its rt->from have already
1687 1688
			 * been updated.
			 */
1689
			if (dst_metric_raw(&entry->dst, RTAX_MTU) &&
1690
			    rt6_mtu_change_route_allowed(idev, entry, mtu))
1691
				dst_metric_set(&entry->dst, RTAX_MTU, mtu);
1692
		}
1693
		bucket++;
1694 1695 1696
	}
}

1697 1698
#define RTF_CACHE_GATEWAY	(RTF_GATEWAY | RTF_CACHE)

1699
static void rt6_exceptions_clean_tohost(struct fib6_info *rt,
1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
					struct in6_addr *gateway)
{
	struct rt6_exception_bucket *bucket;
	struct rt6_exception *rt6_ex;
	struct hlist_node *tmp;
	int i;

	if (!rcu_access_pointer(rt->rt6i_exception_bucket))
		return;

	spin_lock_bh(&rt6_exception_lock);
	bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
				     lockdep_is_held(&rt6_exception_lock));

	if (bucket) {
		for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
			hlist_for_each_entry_safe(rt6_ex, tmp,
						  &bucket->chain, hlist) {
				struct rt6_info *entry = rt6_ex->rt6i;

				if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) ==
				    RTF_CACHE_GATEWAY &&
				    ipv6_addr_equal(gateway,
						    &entry->rt6i_gateway)) {
					rt6_remove_exception(bucket, rt6_ex);
				}
			}
			bucket++;
		}
	}

	spin_unlock_bh(&rt6_exception_lock);
}

1734 1735 1736 1737 1738 1739 1740
static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket,
				      struct rt6_exception *rt6_ex,
				      struct fib6_gc_args *gc_args,
				      unsigned long now)
{
	struct rt6_info *rt = rt6_ex->rt6i;

1741 1742 1743 1744 1745 1746
	/* we are pruning and obsoleting aged-out and non gateway exceptions
	 * even if others have still references to them, so that on next
	 * dst_check() such references can be dropped.
	 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when
	 * expired, independently from their aging, as per RFC 8201 section 4
	 */
W
Wei Wang 已提交
1747 1748 1749 1750 1751 1752 1753 1754
	if (!(rt->rt6i_flags & RTF_EXPIRES)) {
		if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) {
			RT6_TRACE("aging clone %p\n", rt);
			rt6_remove_exception(bucket, rt6_ex);
			return;
		}
	} else if (time_after(jiffies, rt->dst.expires)) {
		RT6_TRACE("purging expired route %p\n", rt);
1755 1756
		rt6_remove_exception(bucket, rt6_ex);
		return;
W
Wei Wang 已提交
1757 1758 1759
	}

	if (rt->rt6i_flags & RTF_GATEWAY) {
1760 1761 1762
		struct neighbour *neigh;
		__u8 neigh_flags = 0;

1763 1764
		neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway);
		if (neigh)
1765
			neigh_flags = neigh->flags;
1766

1767 1768 1769 1770 1771 1772 1773
		if (!(neigh_flags & NTF_ROUTER)) {
			RT6_TRACE("purging route %p via non-router but gateway\n",
				  rt);
			rt6_remove_exception(bucket, rt6_ex);
			return;
		}
	}
W
Wei Wang 已提交
1774

1775 1776 1777
	gc_args->more++;
}

1778
void rt6_age_exceptions(struct fib6_info *rt,
1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789
			struct fib6_gc_args *gc_args,
			unsigned long now)
{
	struct rt6_exception_bucket *bucket;
	struct rt6_exception *rt6_ex;
	struct hlist_node *tmp;
	int i;

	if (!rcu_access_pointer(rt->rt6i_exception_bucket))
		return;

1790 1791
	rcu_read_lock_bh();
	spin_lock(&rt6_exception_lock);
1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804
	bucket = rcu_dereference_protected(rt->rt6i_exception_bucket,
				    lockdep_is_held(&rt6_exception_lock));

	if (bucket) {
		for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
			hlist_for_each_entry_safe(rt6_ex, tmp,
						  &bucket->chain, hlist) {
				rt6_age_examine_exception(bucket, rt6_ex,
							  gc_args, now);
			}
			bucket++;
		}
	}
1805 1806
	spin_unlock(&rt6_exception_lock);
	rcu_read_unlock_bh();
1807 1808
}

1809 1810 1811
/* must be called with rcu lock held */
struct fib6_info *fib6_table_lookup(struct net *net, struct fib6_table *table,
				    int oif, struct flowi6 *fl6, int strict)
L
Linus Torvalds 已提交
1812
{
1813
	struct fib6_node *fn, *saved_fn;
1814
	struct fib6_info *f6i;
L
Linus Torvalds 已提交
1815

1816
	fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
1817
	saved_fn = fn;
L
Linus Torvalds 已提交
1818

D
David Ahern 已提交
1819 1820 1821
	if (fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF)
		oif = 0;

M
Martin KaFai Lau 已提交
1822
redo_rt6_select:
1823 1824
	f6i = rt6_select(net, fn, oif, strict);
	if (f6i == net->ipv6.fib6_null_entry) {
M
Martin KaFai Lau 已提交
1825 1826 1827
		fn = fib6_backtrack(fn, &fl6->saddr);
		if (fn)
			goto redo_rt6_select;
1828 1829 1830 1831 1832 1833
		else if (strict & RT6_LOOKUP_F_REACHABLE) {
			/* also consider unreachable route */
			strict &= ~RT6_LOOKUP_F_REACHABLE;
			fn = saved_fn;
			goto redo_rt6_select;
		}
M
Martin KaFai Lau 已提交
1834 1835
	}

1836 1837
	trace_fib6_table_lookup(net, f6i, table, fl6);

1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
	return f6i;
}

struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
			       int oif, struct flowi6 *fl6,
			       const struct sk_buff *skb, int flags)
{
	struct fib6_info *f6i;
	struct rt6_info *rt;
	int strict = 0;

	strict |= flags & RT6_LOOKUP_F_IFACE;
	strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE;
	if (net->ipv6.devconf_all->forwarding == 0)
		strict |= RT6_LOOKUP_F_REACHABLE;

	rcu_read_lock();

	f6i = fib6_table_lookup(net, table, oif, fl6, strict);
	if (f6i->fib6_nsiblings)
		f6i = fib6_multipath_select(net, f6i, fl6, oif, skb, strict);

1860
	if (f6i == net->ipv6.fib6_null_entry) {
D
David Ahern 已提交
1861
		rt = net->ipv6.ip6_null_entry;
1862
		rcu_read_unlock();
1863 1864
		dst_hold(&rt->dst);
		return rt;
1865 1866 1867 1868 1869
	}

	/*Search through exception table */
	rt = rt6_find_cached_rt(f6i, &fl6->daddr, &fl6->saddr);
	if (rt) {
1870
		if (ip6_hold_safe(net, &rt, true))
1871
			dst_use_noref(&rt->dst, jiffies);
1872

1873
		rcu_read_unlock();
M
Martin KaFai Lau 已提交
1874
		return rt;
1875
	} else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) &&
1876
			    !(f6i->fib6_flags & RTF_GATEWAY))) {
1877 1878 1879 1880 1881 1882 1883
		/* Create a RTF_CACHE clone which will not be
		 * owned by the fib6 tree.  It is for the special case where
		 * the daddr in the skb during the neighbor look-up is different
		 * from the fl6->daddr used to look-up route here.
		 */
		struct rt6_info *uncached_rt;

1884
		uncached_rt = ip6_rt_cache_alloc(f6i, &fl6->daddr, NULL);
1885 1886

		rcu_read_unlock();
T
Thomas Graf 已提交
1887

1888 1889 1890 1891
		if (uncached_rt) {
			/* Uncached_rt's refcnt is taken during ip6_rt_cache_alloc()
			 * No need for another dst_hold()
			 */
1892
			rt6_uncached_list_add(uncached_rt);
W
Wei Wang 已提交
1893
			atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache);
1894
		} else {
1895
			uncached_rt = net->ipv6.ip6_null_entry;
1896 1897
			dst_hold(&uncached_rt->dst);
		}
D
David Ahern 已提交
1898

1899
		return uncached_rt;
M
Martin KaFai Lau 已提交
1900 1901 1902 1903 1904
	} else {
		/* Get a percpu copy */

		struct rt6_info *pcpu_rt;

1905
		local_bh_disable();
1906
		pcpu_rt = rt6_get_pcpu_route(f6i);
M
Martin KaFai Lau 已提交
1907

1908 1909 1910
		if (!pcpu_rt)
			pcpu_rt = rt6_make_pcpu_route(net, f6i);

1911 1912
		local_bh_enable();
		rcu_read_unlock();
1913

M
Martin KaFai Lau 已提交
1914 1915
		return pcpu_rt;
	}
L
Linus Torvalds 已提交
1916
}
1917
EXPORT_SYMBOL_GPL(ip6_pol_route);
L
Linus Torvalds 已提交
1918

D
David Ahern 已提交
1919 1920 1921 1922 1923
static struct rt6_info *ip6_pol_route_input(struct net *net,
					    struct fib6_table *table,
					    struct flowi6 *fl6,
					    const struct sk_buff *skb,
					    int flags)
1924
{
D
David Ahern 已提交
1925
	return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags);
1926 1927
}

1928 1929
struct dst_entry *ip6_route_input_lookup(struct net *net,
					 struct net_device *dev,
D
David Ahern 已提交
1930 1931 1932
					 struct flowi6 *fl6,
					 const struct sk_buff *skb,
					 int flags)
1933 1934 1935 1936
{
	if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG)
		flags |= RT6_LOOKUP_F_IFACE;

D
David Ahern 已提交
1937
	return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input);
1938
}
1939
EXPORT_SYMBOL_GPL(ip6_route_input_lookup);
1940

1941
static void ip6_multipath_l3_keys(const struct sk_buff *skb,
1942 1943
				  struct flow_keys *keys,
				  struct flow_keys *flkeys)
1944 1945 1946
{
	const struct ipv6hdr *outer_iph = ipv6_hdr(skb);
	const struct ipv6hdr *key_iph = outer_iph;
1947
	struct flow_keys *_flkeys = flkeys;
1948 1949 1950
	const struct ipv6hdr *inner_iph;
	const struct icmp6hdr *icmph;
	struct ipv6hdr _inner_iph;
1951
	struct icmp6hdr _icmph;
1952 1953 1954 1955

	if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6))
		goto out;

1956 1957 1958 1959 1960
	icmph = skb_header_pointer(skb, skb_transport_offset(skb),
				   sizeof(_icmph), &_icmph);
	if (!icmph)
		goto out;

1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973
	if (icmph->icmp6_type != ICMPV6_DEST_UNREACH &&
	    icmph->icmp6_type != ICMPV6_PKT_TOOBIG &&
	    icmph->icmp6_type != ICMPV6_TIME_EXCEED &&
	    icmph->icmp6_type != ICMPV6_PARAMPROB)
		goto out;

	inner_iph = skb_header_pointer(skb,
				       skb_transport_offset(skb) + sizeof(*icmph),
				       sizeof(_inner_iph), &_inner_iph);
	if (!inner_iph)
		goto out;

	key_iph = inner_iph;
1974
	_flkeys = NULL;
1975
out:
1976 1977 1978 1979 1980 1981 1982 1983
	if (_flkeys) {
		keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src;
		keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst;
		keys->tags.flow_label = _flkeys->tags.flow_label;
		keys->basic.ip_proto = _flkeys->basic.ip_proto;
	} else {
		keys->addrs.v6addrs.src = key_iph->saddr;
		keys->addrs.v6addrs.dst = key_iph->daddr;
1984
		keys->tags.flow_label = ip6_flowlabel(key_iph);
1985 1986
		keys->basic.ip_proto = key_iph->nexthdr;
	}
1987 1988 1989
}

/* if skb is set it will be used and fl6 can be NULL */
1990 1991
u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6,
		       const struct sk_buff *skb, struct flow_keys *flkeys)
1992 1993
{
	struct flow_keys hash_keys;
1994
	u32 mhash;
1995

1996
	switch (ip6_multipath_hash_policy(net)) {
1997 1998 1999 2000 2001 2002 2003 2004
	case 0:
		memset(&hash_keys, 0, sizeof(hash_keys));
		hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
		if (skb) {
			ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
		} else {
			hash_keys.addrs.v6addrs.src = fl6->saddr;
			hash_keys.addrs.v6addrs.dst = fl6->daddr;
2005
			hash_keys.tags.flow_label = flowi6_get_flowlabel(fl6);
2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039
			hash_keys.basic.ip_proto = fl6->flowi6_proto;
		}
		break;
	case 1:
		if (skb) {
			unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
			struct flow_keys keys;

			/* short-circuit if we already have L4 hash present */
			if (skb->l4_hash)
				return skb_get_hash_raw(skb) >> 1;

			memset(&hash_keys, 0, sizeof(hash_keys));

                        if (!flkeys) {
				skb_flow_dissect_flow_keys(skb, &keys, flag);
				flkeys = &keys;
			}
			hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
			hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
			hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
			hash_keys.ports.src = flkeys->ports.src;
			hash_keys.ports.dst = flkeys->ports.dst;
			hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
		} else {
			memset(&hash_keys, 0, sizeof(hash_keys));
			hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
			hash_keys.addrs.v6addrs.src = fl6->saddr;
			hash_keys.addrs.v6addrs.dst = fl6->daddr;
			hash_keys.ports.src = fl6->fl6_sport;
			hash_keys.ports.dst = fl6->fl6_dport;
			hash_keys.basic.ip_proto = fl6->flowi6_proto;
		}
		break;
2040
	}
2041
	mhash = flow_hash_from_keys(&hash_keys);
2042

2043
	return mhash >> 1;
2044 2045
}

T
Thomas Graf 已提交
2046 2047
void ip6_route_input(struct sk_buff *skb)
{
2048
	const struct ipv6hdr *iph = ipv6_hdr(skb);
2049
	struct net *net = dev_net(skb->dev);
2050
	int flags = RT6_LOOKUP_F_HAS_SADDR;
2051
	struct ip_tunnel_info *tun_info;
2052
	struct flowi6 fl6 = {
2053
		.flowi6_iif = skb->dev->ifindex,
2054 2055
		.daddr = iph->daddr,
		.saddr = iph->saddr,
2056
		.flowlabel = ip6_flowinfo(iph),
2057 2058
		.flowi6_mark = skb->mark,
		.flowi6_proto = iph->nexthdr,
T
Thomas Graf 已提交
2059
	};
2060
	struct flow_keys *flkeys = NULL, _flkeys;
2061

2062
	tun_info = skb_tunnel_info(skb);
2063
	if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
2064
		fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id;
2065 2066 2067 2068

	if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys))
		flkeys = &_flkeys;

2069
	if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6))
2070
		fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys);
2071
	skb_dst_drop(skb);
D
David Ahern 已提交
2072 2073
	skb_dst_set(skb,
		    ip6_route_input_lookup(net, skb->dev, &fl6, skb, flags));
T
Thomas Graf 已提交
2074 2075
}

D
David Ahern 已提交
2076 2077 2078 2079 2080
static struct rt6_info *ip6_pol_route_output(struct net *net,
					     struct fib6_table *table,
					     struct flowi6 *fl6,
					     const struct sk_buff *skb,
					     int flags)
L
Linus Torvalds 已提交
2081
{
D
David Ahern 已提交
2082
	return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags);
T
Thomas Graf 已提交
2083 2084
}

2085 2086
struct dst_entry *ip6_route_output_flags(struct net *net, const struct sock *sk,
					 struct flowi6 *fl6, int flags)
T
Thomas Graf 已提交
2087
{
2088
	bool any_src;
T
Thomas Graf 已提交
2089

2090 2091 2092 2093 2094 2095 2096
	if (rt6_need_strict(&fl6->daddr)) {
		struct dst_entry *dst;

		dst = l3mdev_link_scope_lookup(net, fl6);
		if (dst)
			return dst;
	}
D
David Ahern 已提交
2097

2098
	fl6->flowi6_iif = LOOPBACK_IFINDEX;
2099

2100
	any_src = ipv6_addr_any(&fl6->saddr);
2101
	if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) ||
2102
	    (fl6->flowi6_oif && any_src))
2103
		flags |= RT6_LOOKUP_F_IFACE;
T
Thomas Graf 已提交
2104

2105
	if (!any_src)
2106
		flags |= RT6_LOOKUP_F_HAS_SADDR;
2107 2108
	else if (sk)
		flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs);
2109

D
David Ahern 已提交
2110
	return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output);
L
Linus Torvalds 已提交
2111
}
2112
EXPORT_SYMBOL_GPL(ip6_route_output_flags);
L
Linus Torvalds 已提交
2113

2114
struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2115
{
2116
	struct rt6_info *rt, *ort = (struct rt6_info *) dst_orig;
2117
	struct net_device *loopback_dev = net->loopback_dev;
2118 2119
	struct dst_entry *new = NULL;

2120
	rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1,
2121
		       DST_OBSOLETE_DEAD, 0);
2122
	if (rt) {
2123
		rt6_info_init(rt);
W
Wei Wang 已提交
2124
		atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
2125

2126
		new = &rt->dst;
2127
		new->__use = 1;
2128
		new->input = dst_discard;
E
Eric W. Biederman 已提交
2129
		new->output = dst_discard_out;
2130

2131
		dst_copy_metrics(new, &ort->dst);
2132

2133
		rt->rt6i_idev = in6_dev_get(loopback_dev);
A
Alexey Dobriyan 已提交
2134
		rt->rt6i_gateway = ort->rt6i_gateway;
2135
		rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU;
2136 2137 2138 2139 2140 2141 2142

		memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key));
#ifdef CONFIG_IPV6_SUBTREES
		memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key));
#endif
	}

2143 2144
	dst_release(dst_orig);
	return new ? new : ERR_PTR(-ENOMEM);
2145 2146
}

L
Linus Torvalds 已提交
2147 2148 2149 2150
/*
 *	Destination cache support functions
 */

2151
static bool fib6_check(struct fib6_info *f6i, u32 cookie)
2152 2153 2154
{
	u32 rt_cookie = 0;

2155
	if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie)
2156 2157 2158 2159 2160 2161 2162 2163
		return false;

	if (fib6_check_expired(f6i))
		return false;

	return true;
}

2164 2165 2166
static struct dst_entry *rt6_check(struct rt6_info *rt,
				   struct fib6_info *from,
				   u32 cookie)
2167
{
2168
	u32 rt_cookie = 0;
2169

2170
	if ((from && !fib6_get_cookie_safe(from, &rt_cookie)) ||
2171
	    rt_cookie != cookie)
2172 2173 2174 2175 2176 2177 2178 2179
		return NULL;

	if (rt6_check_expired(rt))
		return NULL;

	return &rt->dst;
}

2180 2181 2182
static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
					    struct fib6_info *from,
					    u32 cookie)
2183
{
2184 2185
	if (!__rt6_check_expired(rt) &&
	    rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
2186
	    fib6_check(from, cookie))
2187 2188 2189 2190 2191
		return &rt->dst;
	else
		return NULL;
}

L
Linus Torvalds 已提交
2192 2193
static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie)
{
2194
	struct dst_entry *dst_ret;
2195
	struct fib6_info *from;
L
Linus Torvalds 已提交
2196 2197
	struct rt6_info *rt;

2198 2199 2200
	rt = container_of(dst, struct rt6_info, dst);

	rcu_read_lock();
L
Linus Torvalds 已提交
2201

2202 2203 2204 2205
	/* All IPV6 dsts are created with ->obsolete set to the value
	 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
	 * into this function always.
	 */
2206

2207 2208 2209 2210 2211
	from = rcu_dereference(rt->from);

	if (from && (rt->rt6i_flags & RTF_PCPU ||
	    unlikely(!list_empty(&rt->rt6i_uncached))))
		dst_ret = rt6_dst_from_check(rt, from, cookie);
2212
	else
2213
		dst_ret = rt6_check(rt, from, cookie);
2214 2215 2216 2217

	rcu_read_unlock();

	return dst_ret;
L
Linus Torvalds 已提交
2218 2219 2220 2221 2222 2223 2224
}

static struct dst_entry *ip6_negative_advice(struct dst_entry *dst)
{
	struct rt6_info *rt = (struct rt6_info *) dst;

	if (rt) {
2225
		if (rt->rt6i_flags & RTF_CACHE) {
2226
			rcu_read_lock();
2227
			if (rt6_check_expired(rt)) {
2228
				rt6_remove_exception_rt(rt);
2229 2230
				dst = NULL;
			}
2231
			rcu_read_unlock();
2232
		} else {
L
Linus Torvalds 已提交
2233
			dst_release(dst);
2234 2235
			dst = NULL;
		}
L
Linus Torvalds 已提交
2236
	}
2237
	return dst;
L
Linus Torvalds 已提交
2238 2239 2240 2241 2242 2243
}

static void ip6_link_failure(struct sk_buff *skb)
{
	struct rt6_info *rt;

2244
	icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
L
Linus Torvalds 已提交
2245

E
Eric Dumazet 已提交
2246
	rt = (struct rt6_info *) skb_dst(skb);
L
Linus Torvalds 已提交
2247
	if (rt) {
2248
		rcu_read_lock();
2249
		if (rt->rt6i_flags & RTF_CACHE) {
W
Wei Wang 已提交
2250
			if (dst_hold_safe(&rt->dst))
2251
				rt6_remove_exception_rt(rt);
2252 2253
		} else {
			struct fib6_info *from;
2254 2255
			struct fib6_node *fn;

2256 2257 2258 2259 2260 2261
			from = rcu_dereference(rt->from);
			if (from) {
				fn = rcu_dereference(from->fib6_node);
				if (fn && (rt->rt6i_flags & RTF_DEFAULT))
					fn->fn_sernum = -1;
			}
2262
		}
2263
		rcu_read_unlock();
L
Linus Torvalds 已提交
2264 2265 2266
	}
}

2267 2268
static void rt6_update_expires(struct rt6_info *rt0, int timeout)
{
2269 2270 2271 2272 2273 2274 2275 2276 2277
	if (!(rt0->rt6i_flags & RTF_EXPIRES)) {
		struct fib6_info *from;

		rcu_read_lock();
		from = rcu_dereference(rt0->from);
		if (from)
			rt0->dst.expires = from->expires;
		rcu_read_unlock();
	}
2278 2279 2280 2281 2282

	dst_set_expires(&rt0->dst, timeout);
	rt0->rt6i_flags |= RTF_EXPIRES;
}

2283 2284 2285 2286
static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu)
{
	struct net *net = dev_net(rt->dst.dev);

2287
	dst_metric_set(&rt->dst, RTAX_MTU, mtu);
2288 2289 2290 2291
	rt->rt6i_flags |= RTF_MODIFIED;
	rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires);
}

2292 2293
static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt)
{
2294 2295 2296 2297 2298 2299
	bool from_set;

	rcu_read_lock();
	from_set = !!rcu_dereference(rt->from);
	rcu_read_unlock();

2300
	return !(rt->rt6i_flags & RTF_CACHE) &&
2301
		(rt->rt6i_flags & RTF_PCPU || from_set);
2302 2303
}

2304 2305
static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
				 const struct ipv6hdr *iph, u32 mtu)
L
Linus Torvalds 已提交
2306
{
2307
	const struct in6_addr *daddr, *saddr;
2308
	struct rt6_info *rt6 = (struct rt6_info *)dst;
L
Linus Torvalds 已提交
2309

2310 2311
	if (rt6->rt6i_flags & RTF_LOCAL)
		return;
2312

2313 2314 2315
	if (dst_metric_locked(dst, RTAX_MTU))
		return;

2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326
	if (iph) {
		daddr = &iph->daddr;
		saddr = &iph->saddr;
	} else if (sk) {
		daddr = &sk->sk_v6_daddr;
		saddr = &inet6_sk(sk)->saddr;
	} else {
		daddr = NULL;
		saddr = NULL;
	}
	dst_confirm_neigh(dst, daddr);
2327 2328 2329
	mtu = max_t(u32, mtu, IPV6_MIN_MTU);
	if (mtu >= dst_mtu(dst))
		return;
2330

2331
	if (!rt6_cache_allowed_for_pmtu(rt6)) {
2332
		rt6_do_update_pmtu(rt6, mtu);
2333 2334 2335
		/* update rt6_ex->stamp for cache */
		if (rt6->rt6i_flags & RTF_CACHE)
			rt6_update_exception_stamp_rt(rt6);
2336
	} else if (daddr) {
2337
		struct fib6_info *from;
2338 2339
		struct rt6_info *nrt6;

2340
		rcu_read_lock();
2341 2342
		from = rcu_dereference(rt6->from);
		nrt6 = ip6_rt_cache_alloc(from, daddr, saddr);
2343 2344
		if (nrt6) {
			rt6_do_update_pmtu(nrt6, mtu);
2345
			if (rt6_insert_exception(nrt6, from))
2346
				dst_release_immediate(&nrt6->dst);
2347
		}
2348
		rcu_read_unlock();
L
Linus Torvalds 已提交
2349 2350 2351
	}
}

2352 2353 2354 2355 2356 2357
static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
			       struct sk_buff *skb, u32 mtu)
{
	__ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu);
}

2358
void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu,
2359
		     int oif, u32 mark, kuid_t uid)
2360 2361 2362 2363 2364 2365 2366
{
	const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
	struct dst_entry *dst;
	struct flowi6 fl6;

	memset(&fl6, 0, sizeof(fl6));
	fl6.flowi6_oif = oif;
2367
	fl6.flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark);
2368 2369
	fl6.daddr = iph->daddr;
	fl6.saddr = iph->saddr;
2370
	fl6.flowlabel = ip6_flowinfo(iph);
2371
	fl6.flowi6_uid = uid;
2372 2373 2374

	dst = ip6_route_output(net, NULL, &fl6);
	if (!dst->error)
2375
		__ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu));
2376 2377 2378 2379 2380 2381
	dst_release(dst);
}
EXPORT_SYMBOL_GPL(ip6_update_pmtu);

void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu)
{
2382 2383
	struct dst_entry *dst;

2384
	ip6_update_pmtu(skb, sock_net(sk), mtu,
2385
			sk->sk_bound_dev_if, sk->sk_mark, sk->sk_uid);
2386 2387 2388 2389 2390 2391 2392 2393 2394 2395

	dst = __sk_dst_get(sk);
	if (!dst || !dst->obsolete ||
	    dst->ops->check(dst, inet6_sk(sk)->dst_cookie))
		return;

	bh_lock_sock(sk);
	if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr))
		ip6_datagram_dst_update(sk, false);
	bh_unlock_sock(sk);
2396 2397 2398
}
EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu);

2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415
void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst,
			   const struct flowi6 *fl6)
{
#ifdef CONFIG_IPV6_SUBTREES
	struct ipv6_pinfo *np = inet6_sk(sk);
#endif

	ip6_dst_store(sk, dst,
		      ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ?
		      &sk->sk_v6_daddr : NULL,
#ifdef CONFIG_IPV6_SUBTREES
		      ipv6_addr_equal(&fl6->saddr, &np->saddr) ?
		      &np->saddr :
#endif
		      NULL);
}

2416 2417 2418 2419 2420 2421 2422 2423 2424
/* Handle redirects */
struct ip6rd_flowi {
	struct flowi6 fl6;
	struct in6_addr gateway;
};

static struct rt6_info *__ip6_route_redirect(struct net *net,
					     struct fib6_table *table,
					     struct flowi6 *fl6,
D
David Ahern 已提交
2425
					     const struct sk_buff *skb,
2426 2427 2428
					     int flags)
{
	struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6;
2429
	struct rt6_info *ret = NULL, *rt_cache;
2430
	struct fib6_info *rt;
2431 2432 2433
	struct fib6_node *fn;

	/* Get the "current" route for this destination and
A
Alexander Alemayhu 已提交
2434
	 * check if the redirect has come from appropriate router.
2435 2436 2437 2438 2439 2440 2441 2442
	 *
	 * RFC 4861 specifies that redirects should only be
	 * accepted if they come from the nexthop to the target.
	 * Due to the way the routes are chosen, this notion
	 * is a bit fuzzy and one might need to check all possible
	 * routes.
	 */

2443
	rcu_read_lock();
2444
	fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
2445
restart:
2446
	for_each_fib6_node_rt_rcu(fn) {
2447
		if (rt->fib6_nh.nh_flags & RTNH_F_DEAD)
2448
			continue;
2449
		if (fib6_check_expired(rt))
2450
			continue;
2451
		if (rt->fib6_flags & RTF_REJECT)
2452
			break;
2453
		if (!(rt->fib6_flags & RTF_GATEWAY))
2454
			continue;
2455
		if (fl6->flowi6_oif != rt->fib6_nh.nh_dev->ifindex)
2456
			continue;
2457 2458 2459 2460 2461
		/* rt_cache's gateway might be different from its 'parent'
		 * in the case of an ip redirect.
		 * So we keep searching in the exception table if the gateway
		 * is different.
		 */
2462
		if (!ipv6_addr_equal(&rdfl->gateway, &rt->fib6_nh.nh_gw)) {
2463 2464 2465 2466 2467 2468
			rt_cache = rt6_find_cached_rt(rt,
						      &fl6->daddr,
						      &fl6->saddr);
			if (rt_cache &&
			    ipv6_addr_equal(&rdfl->gateway,
					    &rt_cache->rt6i_gateway)) {
2469
				ret = rt_cache;
2470 2471
				break;
			}
2472
			continue;
2473
		}
2474 2475 2476 2477
		break;
	}

	if (!rt)
D
David Ahern 已提交
2478
		rt = net->ipv6.fib6_null_entry;
2479
	else if (rt->fib6_flags & RTF_REJECT) {
2480
		ret = net->ipv6.ip6_null_entry;
2481 2482 2483
		goto out;
	}

D
David Ahern 已提交
2484
	if (rt == net->ipv6.fib6_null_entry) {
M
Martin KaFai Lau 已提交
2485 2486 2487
		fn = fib6_backtrack(fn, &fl6->saddr);
		if (fn)
			goto restart;
2488
	}
M
Martin KaFai Lau 已提交
2489

2490
out:
2491 2492 2493 2494
	if (ret)
		dst_hold(&ret->dst);
	else
		ret = ip6_create_rt_rcu(rt);
2495

2496
	rcu_read_unlock();
2497

2498
	trace_fib6_table_lookup(net, rt, table, fl6);
2499
	return ret;
2500 2501 2502
};

static struct dst_entry *ip6_route_redirect(struct net *net,
D
David Ahern 已提交
2503 2504 2505
					    const struct flowi6 *fl6,
					    const struct sk_buff *skb,
					    const struct in6_addr *gateway)
2506 2507 2508 2509 2510 2511 2512
{
	int flags = RT6_LOOKUP_F_HAS_SADDR;
	struct ip6rd_flowi rdfl;

	rdfl.fl6 = *fl6;
	rdfl.gateway = *gateway;

D
David Ahern 已提交
2513
	return fib6_rule_lookup(net, &rdfl.fl6, skb,
2514 2515 2516
				flags, __ip6_route_redirect);
}

2517 2518
void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark,
		  kuid_t uid)
2519 2520 2521 2522 2523 2524
{
	const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
	struct dst_entry *dst;
	struct flowi6 fl6;

	memset(&fl6, 0, sizeof(fl6));
2525
	fl6.flowi6_iif = LOOPBACK_IFINDEX;
2526 2527 2528 2529
	fl6.flowi6_oif = oif;
	fl6.flowi6_mark = mark;
	fl6.daddr = iph->daddr;
	fl6.saddr = iph->saddr;
2530
	fl6.flowlabel = ip6_flowinfo(iph);
2531
	fl6.flowi6_uid = uid;
2532

D
David Ahern 已提交
2533
	dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr);
2534
	rt6_do_redirect(dst, NULL, skb);
2535 2536 2537 2538
	dst_release(dst);
}
EXPORT_SYMBOL_GPL(ip6_redirect);

2539 2540 2541 2542 2543 2544 2545 2546 2547
void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif,
			    u32 mark)
{
	const struct ipv6hdr *iph = ipv6_hdr(skb);
	const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb);
	struct dst_entry *dst;
	struct flowi6 fl6;

	memset(&fl6, 0, sizeof(fl6));
2548
	fl6.flowi6_iif = LOOPBACK_IFINDEX;
2549 2550 2551 2552
	fl6.flowi6_oif = oif;
	fl6.flowi6_mark = mark;
	fl6.daddr = msg->dest;
	fl6.saddr = iph->daddr;
2553
	fl6.flowi6_uid = sock_net_uid(net, NULL);
2554

D
David Ahern 已提交
2555
	dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr);
2556
	rt6_do_redirect(dst, NULL, skb);
2557 2558 2559
	dst_release(dst);
}

2560 2561
void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk)
{
2562 2563
	ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark,
		     sk->sk_uid);
2564 2565 2566
}
EXPORT_SYMBOL_GPL(ip6_sk_redirect);

2567
static unsigned int ip6_default_advmss(const struct dst_entry *dst)
L
Linus Torvalds 已提交
2568
{
2569 2570 2571 2572
	struct net_device *dev = dst->dev;
	unsigned int mtu = dst_mtu(dst);
	struct net *net = dev_net(dev);

L
Linus Torvalds 已提交
2573 2574
	mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);

2575 2576
	if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss)
		mtu = net->ipv6.sysctl.ip6_rt_min_advmss;
L
Linus Torvalds 已提交
2577 2578

	/*
2579 2580 2581
	 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and
	 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size.
	 * IPV6_MAXPLEN is also valid and means: "any MSS,
L
Linus Torvalds 已提交
2582 2583 2584 2585 2586 2587 2588
	 * rely only on pmtu discovery"
	 */
	if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr))
		mtu = IPV6_MAXPLEN;
	return mtu;
}

2589
static unsigned int ip6_mtu(const struct dst_entry *dst)
2590 2591
{
	struct inet6_dev *idev;
2592
	unsigned int mtu;
2593 2594

	mtu = dst_metric_raw(dst, RTAX_MTU);
2595
	if (mtu)
E
Eric Dumazet 已提交
2596
		goto out;
2597 2598

	mtu = IPV6_MIN_MTU;
2599 2600 2601 2602 2603 2604 2605

	rcu_read_lock();
	idev = __in6_dev_get(dst->dev);
	if (idev)
		mtu = idev->cnf.mtu6;
	rcu_read_unlock();

E
Eric Dumazet 已提交
2606
out:
2607 2608 2609
	mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);

	return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
2610 2611
}

2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659
/* MTU selection:
 * 1. mtu on route is locked - use it
 * 2. mtu from nexthop exception
 * 3. mtu from egress device
 *
 * based on ip6_dst_mtu_forward and exception logic of
 * rt6_find_cached_rt; called with rcu_read_lock
 */
u32 ip6_mtu_from_fib6(struct fib6_info *f6i, struct in6_addr *daddr,
		      struct in6_addr *saddr)
{
	struct rt6_exception_bucket *bucket;
	struct rt6_exception *rt6_ex;
	struct in6_addr *src_key;
	struct inet6_dev *idev;
	u32 mtu = 0;

	if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) {
		mtu = f6i->fib6_pmtu;
		if (mtu)
			goto out;
	}

	src_key = NULL;
#ifdef CONFIG_IPV6_SUBTREES
	if (f6i->fib6_src.plen)
		src_key = saddr;
#endif

	bucket = rcu_dereference(f6i->rt6i_exception_bucket);
	rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key);
	if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i))
		mtu = dst_metric_raw(&rt6_ex->rt6i->dst, RTAX_MTU);

	if (likely(!mtu)) {
		struct net_device *dev = fib6_info_nh_dev(f6i);

		mtu = IPV6_MIN_MTU;
		idev = __in6_dev_get(dev);
		if (idev && idev->cnf.mtu6 > mtu)
			mtu = idev->cnf.mtu6;
	}

	mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
out:
	return mtu - lwtunnel_headroom(fib6_info_nh_lwt(f6i), mtu);
}

2660
struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
2661
				  struct flowi6 *fl6)
L
Linus Torvalds 已提交
2662
{
2663
	struct dst_entry *dst;
L
Linus Torvalds 已提交
2664 2665
	struct rt6_info *rt;
	struct inet6_dev *idev = in6_dev_get(dev);
2666
	struct net *net = dev_net(dev);
L
Linus Torvalds 已提交
2667

2668
	if (unlikely(!idev))
E
Eric Dumazet 已提交
2669
		return ERR_PTR(-ENODEV);
L
Linus Torvalds 已提交
2670

2671
	rt = ip6_dst_alloc(net, dev, 0);
2672
	if (unlikely(!rt)) {
L
Linus Torvalds 已提交
2673
		in6_dev_put(idev);
2674
		dst = ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
2675 2676 2677
		goto out;
	}

2678
	rt->dst.flags |= DST_HOST;
2679
	rt->dst.input = ip6_input;
2680
	rt->dst.output  = ip6_output;
2681
	rt->rt6i_gateway  = fl6->daddr;
2682
	rt->rt6i_dst.addr = fl6->daddr;
2683 2684
	rt->rt6i_dst.plen = 128;
	rt->rt6i_idev     = idev;
L
Li RongQing 已提交
2685
	dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0);
L
Linus Torvalds 已提交
2686

2687
	/* Add this dst into uncached_list so that rt6_disable_ip() can
2688 2689 2690
	 * do proper release of the net_device
	 */
	rt6_uncached_list_add(rt);
W
Wei Wang 已提交
2691
	atomic_inc(&net->ipv6.rt6_stats->fib_rt_uncache);
L
Linus Torvalds 已提交
2692

2693 2694
	dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0);

L
Linus Torvalds 已提交
2695
out:
2696
	return dst;
L
Linus Torvalds 已提交
2697 2698
}

2699
static int ip6_dst_gc(struct dst_ops *ops)
L
Linus Torvalds 已提交
2700
{
2701
	struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
2702 2703 2704 2705 2706
	int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
	int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
	int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
	int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
	unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
2707
	int entries;
2708

2709
	entries = dst_entries_get_fast(ops);
2710
	if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
2711
	    entries <= rt_max_size)
L
Linus Torvalds 已提交
2712 2713
		goto out;

2714
	net->ipv6.ip6_rt_gc_expire++;
2715
	fib6_run_gc(net->ipv6.ip6_rt_gc_expire, net, true);
2716 2717
	entries = dst_entries_get_slow(ops);
	if (entries < ops->gc_thresh)
2718
		net->ipv6.ip6_rt_gc_expire = rt_gc_timeout>>1;
L
Linus Torvalds 已提交
2719
out:
2720
	net->ipv6.ip6_rt_gc_expire -= net->ipv6.ip6_rt_gc_expire>>rt_elasticity;
2721
	return entries > rt_max_size;
L
Linus Torvalds 已提交
2722 2723
}

2724
static int ip6_convert_metrics(struct net *net, struct fib6_info *rt,
2725
			       struct fib6_config *cfg)
2726
{
2727
	struct dst_metrics *p;
2728

2729 2730
	if (!cfg->fc_mx)
		return 0;
2731

2732 2733 2734
	p = kzalloc(sizeof(*rt->fib6_metrics), GFP_KERNEL);
	if (unlikely(!p))
		return -ENOMEM;
2735

2736 2737
	refcount_set(&p->refcnt, 1);
	rt->fib6_metrics = p;
2738

2739
	return ip_metrics_convert(net, cfg->fc_mx, cfg->fc_mx_len, p->metrics);
2740
}
L
Linus Torvalds 已提交
2741

2742 2743
static struct rt6_info *ip6_nh_lookup_table(struct net *net,
					    struct fib6_config *cfg,
2744 2745
					    const struct in6_addr *gw_addr,
					    u32 tbid, int flags)
2746 2747 2748 2749 2750 2751 2752 2753 2754
{
	struct flowi6 fl6 = {
		.flowi6_oif = cfg->fc_ifindex,
		.daddr = *gw_addr,
		.saddr = cfg->fc_prefsrc,
	};
	struct fib6_table *table;
	struct rt6_info *rt;

2755
	table = fib6_get_table(net, tbid);
2756 2757 2758 2759 2760 2761
	if (!table)
		return NULL;

	if (!ipv6_addr_any(&cfg->fc_prefsrc))
		flags |= RT6_LOOKUP_F_HAS_SADDR;

2762
	flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE;
D
David Ahern 已提交
2763
	rt = ip6_pol_route(net, table, cfg->fc_ifindex, &fl6, NULL, flags);
2764 2765 2766 2767 2768 2769 2770 2771 2772 2773

	/* if table lookup failed, fall back to full lookup */
	if (rt == net->ipv6.ip6_null_entry) {
		ip6_rt_put(rt);
		rt = NULL;
	}

	return rt;
}

2774 2775
static int ip6_route_check_nh_onlink(struct net *net,
				     struct fib6_config *cfg,
2776
				     const struct net_device *dev,
2777 2778
				     struct netlink_ext_ack *extack)
{
2779
	u32 tbid = l3mdev_fib_table(dev) ? : RT_TABLE_MAIN;
2780 2781 2782 2783 2784 2785 2786 2787
	const struct in6_addr *gw_addr = &cfg->fc_gateway;
	u32 flags = RTF_LOCAL | RTF_ANYCAST | RTF_REJECT;
	struct rt6_info *grt;
	int err;

	err = 0;
	grt = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0);
	if (grt) {
2788 2789
		if (!grt->dst.error &&
		    (grt->rt6i_flags & flags || dev != grt->dst.dev)) {
2790 2791
			NL_SET_ERR_MSG(extack,
				       "Nexthop has invalid gateway or device mismatch");
2792 2793 2794 2795 2796 2797 2798 2799 2800
			err = -EINVAL;
		}

		ip6_rt_put(grt);
	}

	return err;
}

2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
static int ip6_route_check_nh(struct net *net,
			      struct fib6_config *cfg,
			      struct net_device **_dev,
			      struct inet6_dev **idev)
{
	const struct in6_addr *gw_addr = &cfg->fc_gateway;
	struct net_device *dev = _dev ? *_dev : NULL;
	struct rt6_info *grt = NULL;
	int err = -EHOSTUNREACH;

	if (cfg->fc_table) {
2812 2813 2814 2815
		int flags = RT6_LOOKUP_F_IFACE;

		grt = ip6_nh_lookup_table(net, cfg, gw_addr,
					  cfg->fc_table, flags);
2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
		if (grt) {
			if (grt->rt6i_flags & RTF_GATEWAY ||
			    (dev && dev != grt->dst.dev)) {
				ip6_rt_put(grt);
				grt = NULL;
			}
		}
	}

	if (!grt)
D
David Ahern 已提交
2826
		grt = rt6_lookup(net, gw_addr, NULL, cfg->fc_ifindex, NULL, 1);
2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851

	if (!grt)
		goto out;

	if (dev) {
		if (dev != grt->dst.dev) {
			ip6_rt_put(grt);
			goto out;
		}
	} else {
		*_dev = dev = grt->dst.dev;
		*idev = grt->rt6i_idev;
		dev_hold(dev);
		in6_dev_hold(grt->rt6i_idev);
	}

	if (!(grt->rt6i_flags & RTF_GATEWAY))
		err = 0;

	ip6_rt_put(grt);

out:
	return err;
}

2852 2853 2854 2855 2856 2857
static int ip6_validate_gw(struct net *net, struct fib6_config *cfg,
			   struct net_device **_dev, struct inet6_dev **idev,
			   struct netlink_ext_ack *extack)
{
	const struct in6_addr *gw_addr = &cfg->fc_gateway;
	int gwa_type = ipv6_addr_type(gw_addr);
2858
	bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true;
2859
	const struct net_device *dev = *_dev;
2860
	bool need_addr_check = !dev;
2861 2862 2863 2864 2865 2866 2867
	int err = -EINVAL;

	/* if gw_addr is local we will fail to detect this in case
	 * address is still TENTATIVE (DAD in progress). rt6_lookup()
	 * will return already-added prefix route via interface that
	 * prefix route was assigned to, which might be non-loopback.
	 */
2868 2869 2870
	if (dev &&
	    ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
		NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909
		goto out;
	}

	if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) {
		/* IPv6 strictly inhibits using not link-local
		 * addresses as nexthop address.
		 * Otherwise, router will not able to send redirects.
		 * It is very good, but in some (rare!) circumstances
		 * (SIT, PtP, NBMA NOARP links) it is handy to allow
		 * some exceptions. --ANK
		 * We allow IPv4-mapped nexthops to support RFC4798-type
		 * addressing
		 */
		if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) {
			NL_SET_ERR_MSG(extack, "Invalid gateway address");
			goto out;
		}

		if (cfg->fc_flags & RTNH_F_ONLINK)
			err = ip6_route_check_nh_onlink(net, cfg, dev, extack);
		else
			err = ip6_route_check_nh(net, cfg, _dev, idev);

		if (err)
			goto out;
	}

	/* reload in case device was changed */
	dev = *_dev;

	err = -EINVAL;
	if (!dev) {
		NL_SET_ERR_MSG(extack, "Egress device not specified");
		goto out;
	} else if (dev->flags & IFF_LOOPBACK) {
		NL_SET_ERR_MSG(extack,
			       "Egress device can not be loopback device for this route");
		goto out;
	}
2910 2911 2912 2913 2914 2915 2916 2917 2918 2919

	/* if we did not check gw_addr above, do so now that the
	 * egress device has been resolved.
	 */
	if (need_addr_check &&
	    ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
		NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
		goto out;
	}

2920 2921 2922 2923 2924
	err = 0;
out:
	return err;
}

2925
static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg,
2926
					      gfp_t gfp_flags,
2927
					      struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
2928
{
2929
	struct net *net = cfg->fc_nlinfo.nl_net;
2930
	struct fib6_info *rt = NULL;
L
Linus Torvalds 已提交
2931 2932
	struct net_device *dev = NULL;
	struct inet6_dev *idev = NULL;
T
Thomas Graf 已提交
2933
	struct fib6_table *table;
L
Linus Torvalds 已提交
2934
	int addr_type;
2935
	int err = -EINVAL;
L
Linus Torvalds 已提交
2936

2937
	/* RTF_PCPU is an internal flag; can not be set by userspace */
2938 2939
	if (cfg->fc_flags & RTF_PCPU) {
		NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU");
2940
		goto out;
2941
	}
2942

2943 2944 2945 2946 2947 2948
	/* RTF_CACHE is an internal flag; can not be set by userspace */
	if (cfg->fc_flags & RTF_CACHE) {
		NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE");
		goto out;
	}

2949 2950 2951 2952 2953
	if (cfg->fc_type > RTN_MAX) {
		NL_SET_ERR_MSG(extack, "Invalid route type");
		goto out;
	}

2954 2955 2956 2957 2958 2959
	if (cfg->fc_dst_len > 128) {
		NL_SET_ERR_MSG(extack, "Invalid prefix length");
		goto out;
	}
	if (cfg->fc_src_len > 128) {
		NL_SET_ERR_MSG(extack, "Invalid source address length");
2960
		goto out;
2961
	}
L
Linus Torvalds 已提交
2962
#ifndef CONFIG_IPV6_SUBTREES
2963 2964 2965
	if (cfg->fc_src_len) {
		NL_SET_ERR_MSG(extack,
			       "Specifying source address requires IPV6_SUBTREES to be enabled");
2966
		goto out;
2967
	}
L
Linus Torvalds 已提交
2968
#endif
2969
	if (cfg->fc_ifindex) {
L
Linus Torvalds 已提交
2970
		err = -ENODEV;
2971
		dev = dev_get_by_index(net, cfg->fc_ifindex);
L
Linus Torvalds 已提交
2972 2973 2974 2975 2976 2977 2978
		if (!dev)
			goto out;
		idev = in6_dev_get(dev);
		if (!idev)
			goto out;
	}

2979 2980
	if (cfg->fc_metric == 0)
		cfg->fc_metric = IP6_RT_PRIO_USER;
L
Linus Torvalds 已提交
2981

2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996
	if (cfg->fc_flags & RTNH_F_ONLINK) {
		if (!dev) {
			NL_SET_ERR_MSG(extack,
				       "Nexthop device required for onlink");
			err = -ENODEV;
			goto out;
		}

		if (!(dev->flags & IFF_UP)) {
			NL_SET_ERR_MSG(extack, "Nexthop device is not up");
			err = -ENETDOWN;
			goto out;
		}
	}

2997
	err = -ENOBUFS;
2998 2999
	if (cfg->fc_nlinfo.nlh &&
	    !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) {
3000
		table = fib6_get_table(net, cfg->fc_table);
3001
		if (!table) {
3002
			pr_warn("NLM_F_CREATE should be specified when creating new route\n");
3003 3004 3005 3006 3007
			table = fib6_new_table(net, cfg->fc_table);
		}
	} else {
		table = fib6_new_table(net, cfg->fc_table);
	}
3008 3009

	if (!table)
T
Thomas Graf 已提交
3010 3011
		goto out;

3012 3013 3014
	err = -ENOMEM;
	rt = fib6_info_alloc(gfp_flags);
	if (!rt)
L
Linus Torvalds 已提交
3015
		goto out;
3016 3017 3018

	if (cfg->fc_flags & RTF_ADDRCONF)
		rt->dst_nocount = true;
L
Linus Torvalds 已提交
3019

3020 3021 3022 3023
	err = ip6_convert_metrics(net, rt, cfg);
	if (err < 0)
		goto out;

3024
	if (cfg->fc_flags & RTF_EXPIRES)
3025
		fib6_set_expires(rt, jiffies +
3026 3027
				clock_t_to_jiffies(cfg->fc_expires));
	else
3028
		fib6_clean_expires(rt);
L
Linus Torvalds 已提交
3029

3030 3031
	if (cfg->fc_protocol == RTPROT_UNSPEC)
		cfg->fc_protocol = RTPROT_BOOT;
3032
	rt->fib6_protocol = cfg->fc_protocol;
3033 3034

	addr_type = ipv6_addr_type(&cfg->fc_dst);
L
Linus Torvalds 已提交
3035

3036 3037 3038
	if (cfg->fc_encap) {
		struct lwtunnel_state *lwtstate;

3039
		err = lwtunnel_build_state(cfg->fc_encap_type,
3040
					   cfg->fc_encap, AF_INET6, cfg,
3041
					   &lwtstate, extack);
3042 3043
		if (err)
			goto out;
3044
		rt->fib6_nh.nh_lwtstate = lwtstate_get(lwtstate);
3045 3046
	}

3047 3048 3049
	ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len);
	rt->fib6_dst.plen = cfg->fc_dst_len;
	if (rt->fib6_dst.plen == 128)
3050
		rt->dst_host = true;
3051

L
Linus Torvalds 已提交
3052
#ifdef CONFIG_IPV6_SUBTREES
3053 3054
	ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len);
	rt->fib6_src.plen = cfg->fc_src_len;
L
Linus Torvalds 已提交
3055 3056
#endif

3057
	rt->fib6_metric = cfg->fc_metric;
3058
	rt->fib6_nh.nh_weight = 1;
L
Linus Torvalds 已提交
3059

3060 3061
	rt->fib6_type = cfg->fc_type;

L
Linus Torvalds 已提交
3062 3063 3064
	/* We cannot add true routes via loopback here,
	   they would result in kernel looping; promote them to reject routes
	 */
3065
	if ((cfg->fc_flags & RTF_REJECT) ||
3066 3067 3068
	    (dev && (dev->flags & IFF_LOOPBACK) &&
	     !(addr_type & IPV6_ADDR_LOOPBACK) &&
	     !(cfg->fc_flags & RTF_LOCAL))) {
L
Linus Torvalds 已提交
3069
		/* hold loopback dev/idev if we haven't done so. */
3070
		if (dev != net->loopback_dev) {
L
Linus Torvalds 已提交
3071 3072 3073 3074
			if (dev) {
				dev_put(dev);
				in6_dev_put(idev);
			}
3075
			dev = net->loopback_dev;
L
Linus Torvalds 已提交
3076 3077 3078 3079 3080 3081 3082
			dev_hold(dev);
			idev = in6_dev_get(dev);
			if (!idev) {
				err = -ENODEV;
				goto out;
			}
		}
3083
		rt->fib6_flags = RTF_REJECT|RTF_NONEXTHOP;
L
Linus Torvalds 已提交
3084 3085 3086
		goto install_route;
	}

3087
	if (cfg->fc_flags & RTF_GATEWAY) {
3088 3089
		err = ip6_validate_gw(net, cfg, &dev, &idev, extack);
		if (err)
3090
			goto out;
L
Linus Torvalds 已提交
3091

3092
		rt->fib6_nh.nh_gw = cfg->fc_gateway;
L
Linus Torvalds 已提交
3093 3094 3095
	}

	err = -ENODEV;
3096
	if (!dev)
L
Linus Torvalds 已提交
3097 3098
		goto out;

3099 3100 3101 3102 3103 3104
	if (idev->cnf.disable_ipv6) {
		NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device");
		err = -EACCES;
		goto out;
	}

3105 3106 3107 3108 3109 3110
	if (!(dev->flags & IFF_UP)) {
		NL_SET_ERR_MSG(extack, "Nexthop device is not up");
		err = -ENETDOWN;
		goto out;
	}

3111 3112
	if (!ipv6_addr_any(&cfg->fc_prefsrc)) {
		if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) {
3113
			NL_SET_ERR_MSG(extack, "Invalid source address");
3114 3115 3116
			err = -EINVAL;
			goto out;
		}
3117 3118
		rt->fib6_prefsrc.addr = cfg->fc_prefsrc;
		rt->fib6_prefsrc.plen = 128;
3119
	} else
3120
		rt->fib6_prefsrc.plen = 0;
3121

3122
	rt->fib6_flags = cfg->fc_flags;
L
Linus Torvalds 已提交
3123 3124

install_route:
3125
	if (!(rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) &&
3126
	    !netif_carrier_ok(dev))
3127 3128
		rt->fib6_nh.nh_flags |= RTNH_F_LINKDOWN;
	rt->fib6_nh.nh_flags |= (cfg->fc_flags & RTNH_F_ONLINK);
3129
	rt->fib6_nh.nh_dev = dev;
3130
	rt->fib6_table = table;
3131

3132
	cfg->fc_nlinfo.nl_net = dev_net(dev);
3133

D
David Ahern 已提交
3134 3135 3136
	if (idev)
		in6_dev_put(idev);

3137
	return rt;
3138 3139 3140 3141 3142 3143
out:
	if (dev)
		dev_put(dev);
	if (idev)
		in6_dev_put(idev);

3144
	fib6_info_release(rt);
3145
	return ERR_PTR(err);
3146 3147
}

3148 3149
int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags,
		  struct netlink_ext_ack *extack)
3150
{
3151
	struct fib6_info *rt;
3152 3153
	int err;

3154
	rt = ip6_route_info_create(cfg, gfp_flags, extack);
3155 3156
	if (IS_ERR(rt))
		return PTR_ERR(rt);
3157

3158
	err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack);
3159
	fib6_info_release(rt);
3160

L
Linus Torvalds 已提交
3161 3162 3163
	return err;
}

3164
static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info)
L
Linus Torvalds 已提交
3165
{
3166
	struct net *net = info->nl_net;
T
Thomas Graf 已提交
3167
	struct fib6_table *table;
3168
	int err;
L
Linus Torvalds 已提交
3169

D
David Ahern 已提交
3170
	if (rt == net->ipv6.fib6_null_entry) {
3171 3172 3173
		err = -ENOENT;
		goto out;
	}
3174

3175
	table = rt->fib6_table;
3176
	spin_lock_bh(&table->tb6_lock);
3177
	err = fib6_del(rt, info);
3178
	spin_unlock_bh(&table->tb6_lock);
L
Linus Torvalds 已提交
3179

3180
out:
3181
	fib6_info_release(rt);
L
Linus Torvalds 已提交
3182 3183 3184
	return err;
}

3185
int ip6_del_rt(struct net *net, struct fib6_info *rt)
3186
{
3187 3188
	struct nl_info info = { .nl_net = net };

3189
	return __ip6_del_rt(rt, &info);
3190 3191
}

3192
static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg)
3193 3194
{
	struct nl_info *info = &cfg->fc_nlinfo;
3195
	struct net *net = info->nl_net;
3196
	struct sk_buff *skb = NULL;
3197
	struct fib6_table *table;
3198
	int err = -ENOENT;
3199

D
David Ahern 已提交
3200
	if (rt == net->ipv6.fib6_null_entry)
3201
		goto out_put;
3202
	table = rt->fib6_table;
3203
	spin_lock_bh(&table->tb6_lock);
3204

3205
	if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) {
3206
		struct fib6_info *sibling, *next_sibling;
3207

3208 3209 3210 3211 3212
		/* prefer to send a single notification with all hops */
		skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
		if (skb) {
			u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;

3213
			if (rt6_fill_node(net, skb, rt, NULL,
3214 3215 3216 3217 3218 3219 3220 3221
					  NULL, NULL, 0, RTM_DELROUTE,
					  info->portid, seq, 0) < 0) {
				kfree_skb(skb);
				skb = NULL;
			} else
				info->skip_notify = 1;
		}

3222
		list_for_each_entry_safe(sibling, next_sibling,
3223 3224
					 &rt->fib6_siblings,
					 fib6_siblings) {
3225 3226
			err = fib6_del(sibling, info);
			if (err)
3227
				goto out_unlock;
3228 3229 3230 3231
		}
	}

	err = fib6_del(rt, info);
3232
out_unlock:
3233
	spin_unlock_bh(&table->tb6_lock);
3234
out_put:
3235
	fib6_info_release(rt);
3236 3237

	if (skb) {
3238
		rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
3239 3240
			    info->nlh, gfp_any());
	}
3241 3242 3243
	return err;
}

3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259
static int ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg)
{
	int rc = -ESRCH;

	if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex)
		goto out;

	if (cfg->fc_flags & RTF_GATEWAY &&
	    !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway))
		goto out;
	if (dst_hold_safe(&rt->dst))
		rc = rt6_remove_exception_rt(rt);
out:
	return rc;
}

3260 3261
static int ip6_route_del(struct fib6_config *cfg,
			 struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
3262
{
3263
	struct rt6_info *rt_cache;
T
Thomas Graf 已提交
3264
	struct fib6_table *table;
3265
	struct fib6_info *rt;
L
Linus Torvalds 已提交
3266 3267 3268
	struct fib6_node *fn;
	int err = -ESRCH;

3269
	table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table);
3270 3271
	if (!table) {
		NL_SET_ERR_MSG(extack, "FIB table does not exist");
T
Thomas Graf 已提交
3272
		return err;
3273
	}
T
Thomas Graf 已提交
3274

3275
	rcu_read_lock();
L
Linus Torvalds 已提交
3276

T
Thomas Graf 已提交
3277
	fn = fib6_locate(&table->tb6_root,
3278
			 &cfg->fc_dst, cfg->fc_dst_len,
3279
			 &cfg->fc_src, cfg->fc_src_len,
3280
			 !(cfg->fc_flags & RTF_CACHE));
3281

L
Linus Torvalds 已提交
3282
	if (fn) {
3283
		for_each_fib6_node_rt_rcu(fn) {
3284
			if (cfg->fc_flags & RTF_CACHE) {
3285 3286
				int rc;

3287 3288
				rt_cache = rt6_find_cached_rt(rt, &cfg->fc_dst,
							      &cfg->fc_src);
3289 3290
				if (rt_cache) {
					rc = ip6_del_cached_rt(rt_cache, cfg);
3291 3292
					if (rc != -ESRCH) {
						rcu_read_unlock();
3293
						return rc;
3294
					}
3295 3296
				}
				continue;
3297
			}
3298
			if (cfg->fc_ifindex &&
3299 3300
			    (!rt->fib6_nh.nh_dev ||
			     rt->fib6_nh.nh_dev->ifindex != cfg->fc_ifindex))
L
Linus Torvalds 已提交
3301
				continue;
3302
			if (cfg->fc_flags & RTF_GATEWAY &&
3303
			    !ipv6_addr_equal(&cfg->fc_gateway, &rt->fib6_nh.nh_gw))
L
Linus Torvalds 已提交
3304
				continue;
3305
			if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric)
L
Linus Torvalds 已提交
3306
				continue;
3307
			if (cfg->fc_protocol && cfg->fc_protocol != rt->fib6_protocol)
3308
				continue;
3309
			fib6_info_hold(rt);
3310
			rcu_read_unlock();
L
Linus Torvalds 已提交
3311

3312 3313 3314 3315 3316
			/* if gateway was specified only delete the one hop */
			if (cfg->fc_flags & RTF_GATEWAY)
				return __ip6_del_rt(rt, &cfg->fc_nlinfo);

			return __ip6_del_rt_siblings(rt, cfg);
L
Linus Torvalds 已提交
3317 3318
		}
	}
3319
	rcu_read_unlock();
L
Linus Torvalds 已提交
3320 3321 3322 3323

	return err;
}

3324
static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
3325 3326
{
	struct netevent_redirect netevent;
3327 3328 3329 3330
	struct rt6_info *rt, *nrt = NULL;
	struct ndisc_options ndopts;
	struct inet6_dev *in6_dev;
	struct neighbour *neigh;
3331
	struct fib6_info *from;
3332
	struct rd_msg *msg;
3333 3334
	int optlen, on_link;
	u8 *lladdr;
3335

3336
	optlen = skb_tail_pointer(skb) - skb_transport_header(skb);
3337
	optlen -= sizeof(*msg);
3338 3339

	if (optlen < 0) {
3340
		net_dbg_ratelimited("rt6_do_redirect: packet too short\n");
3341 3342 3343
		return;
	}

3344
	msg = (struct rd_msg *)icmp6_hdr(skb);
3345

3346
	if (ipv6_addr_is_multicast(&msg->dest)) {
3347
		net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n");
3348 3349 3350
		return;
	}

3351
	on_link = 0;
3352
	if (ipv6_addr_equal(&msg->dest, &msg->target)) {
3353
		on_link = 1;
3354
	} else if (ipv6_addr_type(&msg->target) !=
3355
		   (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) {
3356
		net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n");
3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370
		return;
	}

	in6_dev = __in6_dev_get(skb->dev);
	if (!in6_dev)
		return;
	if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects)
		return;

	/* RFC2461 8.1:
	 *	The IP source address of the Redirect MUST be the same as the current
	 *	first-hop router for the specified ICMP Destination Address.
	 */

3371
	if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) {
3372 3373 3374
		net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
		return;
	}
3375 3376

	lladdr = NULL;
3377 3378 3379 3380 3381 3382 3383 3384 3385
	if (ndopts.nd_opts_tgt_lladdr) {
		lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr,
					     skb->dev);
		if (!lladdr) {
			net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n");
			return;
		}
	}

3386
	rt = (struct rt6_info *) dst;
3387
	if (rt->rt6i_flags & RTF_REJECT) {
3388
		net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n");
3389
		return;
3390
	}
3391

3392 3393 3394 3395
	/* Redirect received -> path was valid.
	 * Look, redirects are sent only in response to data packets,
	 * so that this nexthop apparently is reachable. --ANK
	 */
3396
	dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr);
3397

3398
	neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1);
3399 3400
	if (!neigh)
		return;
3401

L
Linus Torvalds 已提交
3402 3403 3404 3405
	/*
	 *	We have finally decided to accept it.
	 */

3406
	ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
L
Linus Torvalds 已提交
3407 3408 3409
		     NEIGH_UPDATE_F_WEAK_OVERRIDE|
		     NEIGH_UPDATE_F_OVERRIDE|
		     (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
3410 3411
				     NEIGH_UPDATE_F_ISROUTER)),
		     NDISC_REDIRECT, &ndopts);
L
Linus Torvalds 已提交
3412

3413
	rcu_read_lock();
3414
	from = rcu_dereference(rt->from);
3415
	fib6_info_hold(from);
3416
	rcu_read_unlock();
3417 3418

	nrt = ip6_rt_cache_alloc(from, &msg->dest, NULL);
3419
	if (!nrt)
L
Linus Torvalds 已提交
3420 3421 3422 3423 3424 3425
		goto out;

	nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE;
	if (on_link)
		nrt->rt6i_flags &= ~RTF_GATEWAY;

A
Alexey Dobriyan 已提交
3426
	nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key;
L
Linus Torvalds 已提交
3427

3428 3429 3430 3431
	/* No need to remove rt from the exception table if rt is
	 * a cached route because rt6_insert_exception() will
	 * takes care of it
	 */
3432
	if (rt6_insert_exception(nrt, from)) {
3433 3434 3435
		dst_release_immediate(&nrt->dst);
		goto out;
	}
L
Linus Torvalds 已提交
3436

3437 3438
	netevent.old = &rt->dst;
	netevent.new = &nrt->dst;
3439
	netevent.daddr = &msg->dest;
3440
	netevent.neigh = neigh;
3441 3442
	call_netevent_notifiers(NETEVENT_REDIRECT, &netevent);

L
Linus Torvalds 已提交
3443
out:
3444
	fib6_info_release(from);
3445
	neigh_release(neigh);
3446 3447
}

3448
#ifdef CONFIG_IPV6_ROUTE_INFO
3449
static struct fib6_info *rt6_get_route_info(struct net *net,
3450
					   const struct in6_addr *prefix, int prefixlen,
3451 3452
					   const struct in6_addr *gwaddr,
					   struct net_device *dev)
3453
{
3454 3455
	u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
	int ifindex = dev->ifindex;
3456
	struct fib6_node *fn;
3457
	struct fib6_info *rt = NULL;
T
Thomas Graf 已提交
3458 3459
	struct fib6_table *table;

3460
	table = fib6_get_table(net, tb_id);
3461
	if (!table)
T
Thomas Graf 已提交
3462
		return NULL;
3463

3464
	rcu_read_lock();
3465
	fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true);
3466 3467 3468
	if (!fn)
		goto out;

3469
	for_each_fib6_node_rt_rcu(fn) {
3470
		if (rt->fib6_nh.nh_dev->ifindex != ifindex)
3471
			continue;
3472
		if ((rt->fib6_flags & (RTF_ROUTEINFO|RTF_GATEWAY)) != (RTF_ROUTEINFO|RTF_GATEWAY))
3473
			continue;
3474
		if (!ipv6_addr_equal(&rt->fib6_nh.nh_gw, gwaddr))
3475
			continue;
3476
		fib6_info_hold(rt);
3477 3478 3479
		break;
	}
out:
3480
	rcu_read_unlock();
3481 3482 3483
	return rt;
}

3484
static struct fib6_info *rt6_add_route_info(struct net *net,
3485
					   const struct in6_addr *prefix, int prefixlen,
3486 3487
					   const struct in6_addr *gwaddr,
					   struct net_device *dev,
3488
					   unsigned int pref)
3489
{
3490
	struct fib6_config cfg = {
3491
		.fc_metric	= IP6_RT_PRIO_USER,
3492
		.fc_ifindex	= dev->ifindex,
3493 3494 3495
		.fc_dst_len	= prefixlen,
		.fc_flags	= RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO |
				  RTF_UP | RTF_PREF(pref),
3496
		.fc_protocol = RTPROT_RA,
3497
		.fc_type = RTN_UNICAST,
3498
		.fc_nlinfo.portid = 0,
3499 3500
		.fc_nlinfo.nlh = NULL,
		.fc_nlinfo.nl_net = net,
3501 3502
	};

3503
	cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO,
A
Alexey Dobriyan 已提交
3504 3505
	cfg.fc_dst = *prefix;
	cfg.fc_gateway = *gwaddr;
3506

3507 3508
	/* We should treat it as a default route if prefix length is 0. */
	if (!prefixlen)
3509
		cfg.fc_flags |= RTF_DEFAULT;
3510

3511
	ip6_route_add(&cfg, GFP_ATOMIC, NULL);
3512

3513
	return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev);
3514 3515 3516
}
#endif

3517
struct fib6_info *rt6_get_dflt_router(struct net *net,
3518 3519
				     const struct in6_addr *addr,
				     struct net_device *dev)
3520
{
3521
	u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT;
3522
	struct fib6_info *rt;
T
Thomas Graf 已提交
3523
	struct fib6_table *table;
L
Linus Torvalds 已提交
3524

3525
	table = fib6_get_table(net, tb_id);
3526
	if (!table)
T
Thomas Graf 已提交
3527
		return NULL;
L
Linus Torvalds 已提交
3528

3529 3530
	rcu_read_lock();
	for_each_fib6_node_rt_rcu(&table->tb6_root) {
3531
		if (dev == rt->fib6_nh.nh_dev &&
3532
		    ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) &&
3533
		    ipv6_addr_equal(&rt->fib6_nh.nh_gw, addr))
L
Linus Torvalds 已提交
3534 3535 3536
			break;
	}
	if (rt)
3537
		fib6_info_hold(rt);
3538
	rcu_read_unlock();
L
Linus Torvalds 已提交
3539 3540 3541
	return rt;
}

3542
struct fib6_info *rt6_add_dflt_router(struct net *net,
3543
				     const struct in6_addr *gwaddr,
3544 3545
				     struct net_device *dev,
				     unsigned int pref)
L
Linus Torvalds 已提交
3546
{
3547
	struct fib6_config cfg = {
D
David Ahern 已提交
3548
		.fc_table	= l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT,
3549
		.fc_metric	= IP6_RT_PRIO_USER,
3550 3551 3552
		.fc_ifindex	= dev->ifindex,
		.fc_flags	= RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT |
				  RTF_UP | RTF_EXPIRES | RTF_PREF(pref),
3553
		.fc_protocol = RTPROT_RA,
3554
		.fc_type = RTN_UNICAST,
3555
		.fc_nlinfo.portid = 0,
3556
		.fc_nlinfo.nlh = NULL,
3557
		.fc_nlinfo.nl_net = net,
3558
	};
L
Linus Torvalds 已提交
3559

A
Alexey Dobriyan 已提交
3560
	cfg.fc_gateway = *gwaddr;
L
Linus Torvalds 已提交
3561

3562
	if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) {
3563 3564 3565 3566 3567 3568
		struct fib6_table *table;

		table = fib6_get_table(dev_net(dev), cfg.fc_table);
		if (table)
			table->flags |= RT6_TABLE_HAS_DFLT_ROUTER;
	}
L
Linus Torvalds 已提交
3569

3570
	return rt6_get_dflt_router(net, gwaddr, dev);
L
Linus Torvalds 已提交
3571 3572
}

3573 3574
static void __rt6_purge_dflt_routers(struct net *net,
				     struct fib6_table *table)
L
Linus Torvalds 已提交
3575
{
3576
	struct fib6_info *rt;
L
Linus Torvalds 已提交
3577 3578

restart:
3579 3580
	rcu_read_lock();
	for_each_fib6_node_rt_rcu(&table->tb6_root) {
D
David Ahern 已提交
3581 3582 3583
		struct net_device *dev = fib6_info_nh_dev(rt);
		struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;

3584
		if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
D
David Ahern 已提交
3585
		    (!idev || idev->cnf.accept_ra != 2)) {
3586 3587 3588
			fib6_info_hold(rt);
			rcu_read_unlock();
			ip6_del_rt(net, rt);
L
Linus Torvalds 已提交
3589 3590 3591
			goto restart;
		}
	}
3592
	rcu_read_unlock();
3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608

	table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER;
}

void rt6_purge_dflt_routers(struct net *net)
{
	struct fib6_table *table;
	struct hlist_head *head;
	unsigned int h;

	rcu_read_lock();

	for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
		head = &net->ipv6.fib_table_hash[h];
		hlist_for_each_entry_rcu(table, head, tb6_hlist) {
			if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER)
3609
				__rt6_purge_dflt_routers(net, table);
3610 3611 3612 3613
		}
	}

	rcu_read_unlock();
L
Linus Torvalds 已提交
3614 3615
}

3616 3617
static void rtmsg_to_fib6_config(struct net *net,
				 struct in6_rtmsg *rtmsg,
3618 3619 3620 3621
				 struct fib6_config *cfg)
{
	memset(cfg, 0, sizeof(*cfg));

D
David Ahern 已提交
3622 3623
	cfg->fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ?
			 : RT6_TABLE_MAIN;
3624 3625 3626 3627 3628 3629
	cfg->fc_ifindex = rtmsg->rtmsg_ifindex;
	cfg->fc_metric = rtmsg->rtmsg_metric;
	cfg->fc_expires = rtmsg->rtmsg_info;
	cfg->fc_dst_len = rtmsg->rtmsg_dst_len;
	cfg->fc_src_len = rtmsg->rtmsg_src_len;
	cfg->fc_flags = rtmsg->rtmsg_flags;
3630
	cfg->fc_type = rtmsg->rtmsg_type;
3631

3632
	cfg->fc_nlinfo.nl_net = net;
3633

A
Alexey Dobriyan 已提交
3634 3635 3636
	cfg->fc_dst = rtmsg->rtmsg_dst;
	cfg->fc_src = rtmsg->rtmsg_src;
	cfg->fc_gateway = rtmsg->rtmsg_gateway;
3637 3638
}

3639
int ipv6_route_ioctl(struct net *net, unsigned int cmd, void __user *arg)
L
Linus Torvalds 已提交
3640
{
3641
	struct fib6_config cfg;
L
Linus Torvalds 已提交
3642 3643 3644
	struct in6_rtmsg rtmsg;
	int err;

3645
	switch (cmd) {
L
Linus Torvalds 已提交
3646 3647
	case SIOCADDRT:		/* Add a route */
	case SIOCDELRT:		/* Delete a route */
3648
		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
L
Linus Torvalds 已提交
3649 3650 3651 3652 3653
			return -EPERM;
		err = copy_from_user(&rtmsg, arg,
				     sizeof(struct in6_rtmsg));
		if (err)
			return -EFAULT;
3654

3655
		rtmsg_to_fib6_config(net, &rtmsg, &cfg);
3656

L
Linus Torvalds 已提交
3657 3658 3659
		rtnl_lock();
		switch (cmd) {
		case SIOCADDRT:
3660
			err = ip6_route_add(&cfg, GFP_KERNEL, NULL);
L
Linus Torvalds 已提交
3661 3662
			break;
		case SIOCDELRT:
3663
			err = ip6_route_del(&cfg, NULL);
L
Linus Torvalds 已提交
3664 3665 3666 3667 3668 3669 3670
			break;
		default:
			err = -EINVAL;
		}
		rtnl_unlock();

		return err;
3671
	}
L
Linus Torvalds 已提交
3672 3673 3674 3675 3676 3677 3678 3679

	return -EINVAL;
}

/*
 *	Drop the packet on the floor
 */

3680
static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes)
L
Linus Torvalds 已提交
3681
{
3682
	int type;
E
Eric Dumazet 已提交
3683
	struct dst_entry *dst = skb_dst(skb);
3684 3685
	switch (ipstats_mib_noroutes) {
	case IPSTATS_MIB_INNOROUTES:
3686
		type = ipv6_addr_type(&ipv6_hdr(skb)->daddr);
U
Ulrich Weber 已提交
3687
		if (type == IPV6_ADDR_ANY) {
3688 3689
			IP6_INC_STATS(dev_net(dst->dev),
				      __in6_dev_get_safely(skb->dev),
3690
				      IPSTATS_MIB_INADDRERRORS);
3691 3692 3693 3694
			break;
		}
		/* FALLTHROUGH */
	case IPSTATS_MIB_OUTNOROUTES:
3695 3696
		IP6_INC_STATS(dev_net(dst->dev), ip6_dst_idev(dst),
			      ipstats_mib_noroutes);
3697 3698
		break;
	}
3699
	icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0);
L
Linus Torvalds 已提交
3700 3701 3702 3703
	kfree_skb(skb);
	return 0;
}

3704 3705
static int ip6_pkt_discard(struct sk_buff *skb)
{
3706
	return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES);
3707 3708
}

E
Eric W. Biederman 已提交
3709
static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
3710
{
E
Eric Dumazet 已提交
3711
	skb->dev = skb_dst(skb)->dev;
3712
	return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES);
L
Linus Torvalds 已提交
3713 3714
}

3715 3716
static int ip6_pkt_prohibit(struct sk_buff *skb)
{
3717
	return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES);
3718 3719
}

E
Eric W. Biederman 已提交
3720
static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb)
3721
{
E
Eric Dumazet 已提交
3722
	skb->dev = skb_dst(skb)->dev;
3723
	return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES);
3724 3725
}

L
Linus Torvalds 已提交
3726 3727 3728 3729
/*
 *	Allocate a dst for local (unicast / anycast) address.
 */

3730 3731 3732 3733
struct fib6_info *addrconf_f6i_alloc(struct net *net,
				     struct inet6_dev *idev,
				     const struct in6_addr *addr,
				     bool anycast, gfp_t gfp_flags)
L
Linus Torvalds 已提交
3734
{
D
David Ahern 已提交
3735
	u32 tb_id;
3736
	struct net_device *dev = idev->dev;
3737
	struct fib6_info *f6i;
3738

3739 3740
	f6i = fib6_info_alloc(gfp_flags);
	if (!f6i)
L
Linus Torvalds 已提交
3741 3742
		return ERR_PTR(-ENOMEM);

3743 3744 3745 3746
	f6i->dst_nocount = true;
	f6i->dst_host = true;
	f6i->fib6_protocol = RTPROT_KERNEL;
	f6i->fib6_flags = RTF_UP | RTF_NONEXTHOP;
3747
	if (anycast) {
3748 3749
		f6i->fib6_type = RTN_ANYCAST;
		f6i->fib6_flags |= RTF_ANYCAST;
3750
	} else {
3751 3752
		f6i->fib6_type = RTN_LOCAL;
		f6i->fib6_flags |= RTF_LOCAL;
3753
	}
L
Linus Torvalds 已提交
3754

3755
	f6i->fib6_nh.nh_gw = *addr;
3756
	dev_hold(dev);
3757 3758 3759
	f6i->fib6_nh.nh_dev = dev;
	f6i->fib6_dst.addr = *addr;
	f6i->fib6_dst.plen = 128;
D
David Ahern 已提交
3760
	tb_id = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL;
3761
	f6i->fib6_table = fib6_get_table(net, tb_id);
L
Linus Torvalds 已提交
3762

3763
	return f6i;
L
Linus Torvalds 已提交
3764 3765
}

3766 3767 3768 3769 3770 3771 3772
/* remove deleted ip from prefsrc entries */
struct arg_dev_net_ip {
	struct net_device *dev;
	struct net *net;
	struct in6_addr *addr;
};

3773
static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg)
3774 3775 3776 3777 3778
{
	struct net_device *dev = ((struct arg_dev_net_ip *)arg)->dev;
	struct net *net = ((struct arg_dev_net_ip *)arg)->net;
	struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr;

3779
	if (((void *)rt->fib6_nh.nh_dev == dev || !dev) &&
D
David Ahern 已提交
3780
	    rt != net->ipv6.fib6_null_entry &&
3781
	    ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr)) {
3782
		spin_lock_bh(&rt6_exception_lock);
3783
		/* remove prefsrc entry */
3784
		rt->fib6_prefsrc.plen = 0;
3785 3786 3787
		/* need to update cache as well */
		rt6_exceptions_remove_prefsrc(rt);
		spin_unlock_bh(&rt6_exception_lock);
3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799
	}
	return 0;
}

void rt6_remove_prefsrc(struct inet6_ifaddr *ifp)
{
	struct net *net = dev_net(ifp->idev->dev);
	struct arg_dev_net_ip adni = {
		.dev = ifp->idev->dev,
		.net = net,
		.addr = &ifp->addr,
	};
3800
	fib6_clean_all(net, fib6_remove_prefsrc, &adni);
3801 3802
}

3803 3804 3805
#define RTF_RA_ROUTER		(RTF_ADDRCONF | RTF_DEFAULT | RTF_GATEWAY)

/* Remove routers and update dst entries when gateway turn into host. */
3806
static int fib6_clean_tohost(struct fib6_info *rt, void *arg)
3807 3808 3809
{
	struct in6_addr *gateway = (struct in6_addr *)arg;

3810
	if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) &&
3811
	    ipv6_addr_equal(gateway, &rt->fib6_nh.nh_gw)) {
3812 3813
		return -1;
	}
3814 3815 3816 3817 3818 3819 3820

	/* Further clean up cached routes in exception table.
	 * This is needed because cached route may have a different
	 * gateway than its 'parent' in the case of an ip redirect.
	 */
	rt6_exceptions_clean_tohost(rt, gateway);

3821 3822 3823 3824 3825 3826 3827 3828
	return 0;
}

void rt6_clean_tohost(struct net *net, struct in6_addr *gateway)
{
	fib6_clean_all(net, fib6_clean_tohost, gateway);
}

3829 3830
struct arg_netdev_event {
	const struct net_device *dev;
3831 3832 3833 3834
	union {
		unsigned int nh_flags;
		unsigned long event;
	};
3835 3836
};

3837
static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt)
3838
{
3839
	struct fib6_info *iter;
3840 3841
	struct fib6_node *fn;

3842 3843
	fn = rcu_dereference_protected(rt->fib6_node,
			lockdep_is_held(&rt->fib6_table->tb6_lock));
3844
	iter = rcu_dereference_protected(fn->leaf,
3845
			lockdep_is_held(&rt->fib6_table->tb6_lock));
3846
	while (iter) {
3847
		if (iter->fib6_metric == rt->fib6_metric &&
3848
		    iter->fib6_nsiblings)
3849
			return iter;
3850
		iter = rcu_dereference_protected(iter->fib6_next,
3851
				lockdep_is_held(&rt->fib6_table->tb6_lock));
3852 3853 3854 3855 3856
	}

	return NULL;
}

3857
static bool rt6_is_dead(const struct fib6_info *rt)
3858
{
3859 3860
	if (rt->fib6_nh.nh_flags & RTNH_F_DEAD ||
	    (rt->fib6_nh.nh_flags & RTNH_F_LINKDOWN &&
D
David Ahern 已提交
3861
	     fib6_ignore_linkdown(rt)))
3862 3863 3864 3865 3866
		return true;

	return false;
}

3867
static int rt6_multipath_total_weight(const struct fib6_info *rt)
3868
{
3869
	struct fib6_info *iter;
3870 3871 3872
	int total = 0;

	if (!rt6_is_dead(rt))
3873
		total += rt->fib6_nh.nh_weight;
3874

3875
	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
3876
		if (!rt6_is_dead(iter))
3877
			total += iter->fib6_nh.nh_weight;
3878 3879 3880 3881 3882
	}

	return total;
}

3883
static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total)
3884 3885 3886 3887
{
	int upper_bound = -1;

	if (!rt6_is_dead(rt)) {
3888
		*weight += rt->fib6_nh.nh_weight;
3889 3890 3891
		upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31,
						    total) - 1;
	}
3892
	atomic_set(&rt->fib6_nh.nh_upper_bound, upper_bound);
3893 3894
}

3895
static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total)
3896
{
3897
	struct fib6_info *iter;
3898 3899 3900 3901
	int weight = 0;

	rt6_upper_bound_set(rt, &weight, total);

3902
	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3903 3904 3905
		rt6_upper_bound_set(iter, &weight, total);
}

3906
void rt6_multipath_rebalance(struct fib6_info *rt)
3907
{
3908
	struct fib6_info *first;
3909 3910 3911 3912 3913 3914
	int total;

	/* In case the entire multipath route was marked for flushing,
	 * then there is no need to rebalance upon the removal of every
	 * sibling route.
	 */
3915
	if (!rt->fib6_nsiblings || rt->should_flush)
3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929
		return;

	/* During lookup routes are evaluated in order, so we need to
	 * make sure upper bounds are assigned from the first sibling
	 * onwards.
	 */
	first = rt6_multipath_first_sibling(rt);
	if (WARN_ON_ONCE(!first))
		return;

	total = rt6_multipath_total_weight(first);
	rt6_multipath_upper_bound_set(first, total);
}

3930
static int fib6_ifup(struct fib6_info *rt, void *p_arg)
3931 3932
{
	const struct arg_netdev_event *arg = p_arg;
3933
	struct net *net = dev_net(arg->dev);
3934

D
David Ahern 已提交
3935
	if (rt != net->ipv6.fib6_null_entry && rt->fib6_nh.nh_dev == arg->dev) {
3936
		rt->fib6_nh.nh_flags &= ~arg->nh_flags;
3937
		fib6_update_sernum_upto_root(net, rt);
3938
		rt6_multipath_rebalance(rt);
3939
	}
3940 3941 3942 3943 3944 3945 3946 3947

	return 0;
}

void rt6_sync_up(struct net_device *dev, unsigned int nh_flags)
{
	struct arg_netdev_event arg = {
		.dev = dev,
I
Ido Schimmel 已提交
3948 3949 3950
		{
			.nh_flags = nh_flags,
		},
3951 3952 3953 3954 3955 3956 3957 3958
	};

	if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev))
		arg.nh_flags |= RTNH_F_LINKDOWN;

	fib6_clean_all(dev_net(dev), fib6_ifup, &arg);
}

3959
static bool rt6_multipath_uses_dev(const struct fib6_info *rt,
3960 3961
				   const struct net_device *dev)
{
3962
	struct fib6_info *iter;
3963

3964
	if (rt->fib6_nh.nh_dev == dev)
3965
		return true;
3966
	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3967
		if (iter->fib6_nh.nh_dev == dev)
3968 3969 3970 3971 3972
			return true;

	return false;
}

3973
static void rt6_multipath_flush(struct fib6_info *rt)
3974
{
3975
	struct fib6_info *iter;
3976 3977

	rt->should_flush = 1;
3978
	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3979 3980 3981
		iter->should_flush = 1;
}

3982
static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt,
3983 3984
					     const struct net_device *down_dev)
{
3985
	struct fib6_info *iter;
3986 3987
	unsigned int dead = 0;

3988 3989
	if (rt->fib6_nh.nh_dev == down_dev ||
	    rt->fib6_nh.nh_flags & RTNH_F_DEAD)
3990
		dead++;
3991
	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
3992 3993
		if (iter->fib6_nh.nh_dev == down_dev ||
		    iter->fib6_nh.nh_flags & RTNH_F_DEAD)
3994 3995 3996 3997 3998
			dead++;

	return dead;
}

3999
static void rt6_multipath_nh_flags_set(struct fib6_info *rt,
4000 4001 4002
				       const struct net_device *dev,
				       unsigned int nh_flags)
{
4003
	struct fib6_info *iter;
4004

4005 4006
	if (rt->fib6_nh.nh_dev == dev)
		rt->fib6_nh.nh_flags |= nh_flags;
4007
	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
4008 4009
		if (iter->fib6_nh.nh_dev == dev)
			iter->fib6_nh.nh_flags |= nh_flags;
4010 4011
}

4012
/* called with write lock held for table with rt */
4013
static int fib6_ifdown(struct fib6_info *rt, void *p_arg)
L
Linus Torvalds 已提交
4014
{
4015 4016
	const struct arg_netdev_event *arg = p_arg;
	const struct net_device *dev = arg->dev;
4017
	struct net *net = dev_net(dev);
4018

D
David Ahern 已提交
4019
	if (rt == net->ipv6.fib6_null_entry)
4020 4021 4022 4023
		return 0;

	switch (arg->event) {
	case NETDEV_UNREGISTER:
4024
		return rt->fib6_nh.nh_dev == dev ? -1 : 0;
4025
	case NETDEV_DOWN:
4026
		if (rt->should_flush)
4027
			return -1;
4028
		if (!rt->fib6_nsiblings)
4029
			return rt->fib6_nh.nh_dev == dev ? -1 : 0;
4030 4031 4032 4033
		if (rt6_multipath_uses_dev(rt, dev)) {
			unsigned int count;

			count = rt6_multipath_dead_count(rt, dev);
4034
			if (rt->fib6_nsiblings + 1 == count) {
4035 4036 4037 4038 4039
				rt6_multipath_flush(rt);
				return -1;
			}
			rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD |
						   RTNH_F_LINKDOWN);
4040
			fib6_update_sernum(net, rt);
4041
			rt6_multipath_rebalance(rt);
4042 4043
		}
		return -2;
4044
	case NETDEV_CHANGE:
4045
		if (rt->fib6_nh.nh_dev != dev ||
4046
		    rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
4047
			break;
4048
		rt->fib6_nh.nh_flags |= RTNH_F_LINKDOWN;
4049
		rt6_multipath_rebalance(rt);
4050
		break;
4051
	}
4052

L
Linus Torvalds 已提交
4053 4054 4055
	return 0;
}

4056
void rt6_sync_down_dev(struct net_device *dev, unsigned long event)
L
Linus Torvalds 已提交
4057
{
4058
	struct arg_netdev_event arg = {
4059
		.dev = dev,
I
Ido Schimmel 已提交
4060 4061 4062
		{
			.event = event,
		},
4063 4064
	};

4065 4066 4067 4068 4069 4070 4071 4072
	fib6_clean_all(dev_net(dev), fib6_ifdown, &arg);
}

void rt6_disable_ip(struct net_device *dev, unsigned long event)
{
	rt6_sync_down_dev(dev, event);
	rt6_uncached_list_flush_dev(dev_net(dev), dev);
	neigh_ifdown(&nd_tbl, dev);
L
Linus Torvalds 已提交
4073 4074
}

4075
struct rt6_mtu_change_arg {
L
Linus Torvalds 已提交
4076
	struct net_device *dev;
4077
	unsigned int mtu;
L
Linus Torvalds 已提交
4078 4079
};

4080
static int rt6_mtu_change_route(struct fib6_info *rt, void *p_arg)
L
Linus Torvalds 已提交
4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091
{
	struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg;
	struct inet6_dev *idev;

	/* In IPv6 pmtu discovery is not optional,
	   so that RTAX_MTU lock cannot disable it.
	   We still use this lock to block changes
	   caused by addrconf/ndisc.
	*/

	idev = __in6_dev_get(arg->dev);
4092
	if (!idev)
L
Linus Torvalds 已提交
4093 4094 4095 4096 4097 4098 4099
		return 0;

	/* For administrative MTU increase, there is no way to discover
	   IPv6 PMTU increase, so PMTU increase should be updated here.
	   Since RFC 1981 doesn't include administrative MTU increase
	   update PMTU increase is a MUST. (i.e. jumbo frame)
	 */
4100
	if (rt->fib6_nh.nh_dev == arg->dev &&
4101 4102 4103 4104 4105 4106 4107
	    !fib6_metric_locked(rt, RTAX_MTU)) {
		u32 mtu = rt->fib6_pmtu;

		if (mtu >= arg->mtu ||
		    (mtu < arg->mtu && mtu == idev->cnf.mtu6))
			fib6_metric_set(rt, RTAX_MTU, arg->mtu);

4108
		spin_lock_bh(&rt6_exception_lock);
4109
		rt6_exceptions_update_pmtu(idev, rt, arg->mtu);
4110
		spin_unlock_bh(&rt6_exception_lock);
4111
	}
L
Linus Torvalds 已提交
4112 4113 4114
	return 0;
}

4115
void rt6_mtu_change(struct net_device *dev, unsigned int mtu)
L
Linus Torvalds 已提交
4116
{
T
Thomas Graf 已提交
4117 4118 4119 4120
	struct rt6_mtu_change_arg arg = {
		.dev = dev,
		.mtu = mtu,
	};
L
Linus Torvalds 已提交
4121

4122
	fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg);
L
Linus Torvalds 已提交
4123 4124
}

4125
static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = {
4126
	[RTA_GATEWAY]           = { .len = sizeof(struct in6_addr) },
4127
	[RTA_PREFSRC]		= { .len = sizeof(struct in6_addr) },
4128
	[RTA_OIF]               = { .type = NLA_U32 },
4129
	[RTA_IIF]		= { .type = NLA_U32 },
4130 4131
	[RTA_PRIORITY]          = { .type = NLA_U32 },
	[RTA_METRICS]           = { .type = NLA_NESTED },
4132
	[RTA_MULTIPATH]		= { .len = sizeof(struct rtnexthop) },
4133
	[RTA_PREF]              = { .type = NLA_U8 },
4134 4135
	[RTA_ENCAP_TYPE]	= { .type = NLA_U16 },
	[RTA_ENCAP]		= { .type = NLA_NESTED },
4136
	[RTA_EXPIRES]		= { .type = NLA_U32 },
4137
	[RTA_UID]		= { .type = NLA_U32 },
4138
	[RTA_MARK]		= { .type = NLA_U32 },
4139
	[RTA_TABLE]		= { .type = NLA_U32 },
4140 4141 4142
	[RTA_IP_PROTO]		= { .type = NLA_U8 },
	[RTA_SPORT]		= { .type = NLA_U16 },
	[RTA_DPORT]		= { .type = NLA_U16 },
4143 4144 4145
};

static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
4146 4147
			      struct fib6_config *cfg,
			      struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
4148
{
4149 4150
	struct rtmsg *rtm;
	struct nlattr *tb[RTA_MAX+1];
4151
	unsigned int pref;
4152
	int err;
L
Linus Torvalds 已提交
4153

4154 4155
	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv6_policy,
			  NULL);
4156 4157
	if (err < 0)
		goto errout;
L
Linus Torvalds 已提交
4158

4159 4160 4161 4162 4163 4164 4165 4166 4167
	err = -EINVAL;
	rtm = nlmsg_data(nlh);
	memset(cfg, 0, sizeof(*cfg));

	cfg->fc_table = rtm->rtm_table;
	cfg->fc_dst_len = rtm->rtm_dst_len;
	cfg->fc_src_len = rtm->rtm_src_len;
	cfg->fc_flags = RTF_UP;
	cfg->fc_protocol = rtm->rtm_protocol;
4168
	cfg->fc_type = rtm->rtm_type;
4169

4170 4171
	if (rtm->rtm_type == RTN_UNREACHABLE ||
	    rtm->rtm_type == RTN_BLACKHOLE ||
4172 4173
	    rtm->rtm_type == RTN_PROHIBIT ||
	    rtm->rtm_type == RTN_THROW)
4174 4175
		cfg->fc_flags |= RTF_REJECT;

4176 4177 4178
	if (rtm->rtm_type == RTN_LOCAL)
		cfg->fc_flags |= RTF_LOCAL;

4179 4180 4181
	if (rtm->rtm_flags & RTM_F_CLONED)
		cfg->fc_flags |= RTF_CACHE;

4182 4183
	cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK);

4184
	cfg->fc_nlinfo.portid = NETLINK_CB(skb).portid;
4185
	cfg->fc_nlinfo.nlh = nlh;
4186
	cfg->fc_nlinfo.nl_net = sock_net(skb->sk);
4187 4188

	if (tb[RTA_GATEWAY]) {
4189
		cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]);
4190
		cfg->fc_flags |= RTF_GATEWAY;
L
Linus Torvalds 已提交
4191
	}
4192 4193 4194 4195 4196 4197 4198 4199

	if (tb[RTA_DST]) {
		int plen = (rtm->rtm_dst_len + 7) >> 3;

		if (nla_len(tb[RTA_DST]) < plen)
			goto errout;

		nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen);
L
Linus Torvalds 已提交
4200
	}
4201 4202 4203 4204 4205 4206 4207 4208

	if (tb[RTA_SRC]) {
		int plen = (rtm->rtm_src_len + 7) >> 3;

		if (nla_len(tb[RTA_SRC]) < plen)
			goto errout;

		nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen);
L
Linus Torvalds 已提交
4209
	}
4210

4211
	if (tb[RTA_PREFSRC])
4212
		cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]);
4213

4214 4215 4216 4217 4218 4219 4220 4221 4222
	if (tb[RTA_OIF])
		cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]);

	if (tb[RTA_PRIORITY])
		cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]);

	if (tb[RTA_METRICS]) {
		cfg->fc_mx = nla_data(tb[RTA_METRICS]);
		cfg->fc_mx_len = nla_len(tb[RTA_METRICS]);
L
Linus Torvalds 已提交
4223
	}
4224 4225 4226 4227

	if (tb[RTA_TABLE])
		cfg->fc_table = nla_get_u32(tb[RTA_TABLE]);

4228 4229 4230
	if (tb[RTA_MULTIPATH]) {
		cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]);
		cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]);
4231 4232

		err = lwtunnel_valid_encap_type_attr(cfg->fc_mp,
4233
						     cfg->fc_mp_len, extack);
4234 4235
		if (err < 0)
			goto errout;
4236 4237
	}

4238 4239 4240 4241 4242 4243 4244 4245
	if (tb[RTA_PREF]) {
		pref = nla_get_u8(tb[RTA_PREF]);
		if (pref != ICMPV6_ROUTER_PREF_LOW &&
		    pref != ICMPV6_ROUTER_PREF_HIGH)
			pref = ICMPV6_ROUTER_PREF_MEDIUM;
		cfg->fc_flags |= RTF_PREF(pref);
	}

4246 4247 4248
	if (tb[RTA_ENCAP])
		cfg->fc_encap = tb[RTA_ENCAP];

4249
	if (tb[RTA_ENCAP_TYPE]) {
4250 4251
		cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]);

4252
		err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack);
4253 4254 4255 4256
		if (err < 0)
			goto errout;
	}

4257 4258 4259 4260 4261 4262 4263 4264 4265
	if (tb[RTA_EXPIRES]) {
		unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ);

		if (addrconf_finite_timeout(timeout)) {
			cfg->fc_expires = jiffies_to_clock_t(timeout * HZ);
			cfg->fc_flags |= RTF_EXPIRES;
		}
	}

4266 4267 4268
	err = 0;
errout:
	return err;
L
Linus Torvalds 已提交
4269 4270
}

4271
struct rt6_nh {
4272
	struct fib6_info *fib6_info;
4273 4274 4275 4276 4277 4278 4279 4280 4281
	struct fib6_config r_cfg;
	struct list_head next;
};

static void ip6_print_replace_route_err(struct list_head *rt6_nh_list)
{
	struct rt6_nh *nh;

	list_for_each_entry(nh, rt6_nh_list, next) {
4282
		pr_warn("IPV6: multipath route replace failed (check consistency of installed routes): %pI6c nexthop %pI6c ifi %d\n",
4283 4284 4285 4286 4287
		        &nh->r_cfg.fc_dst, &nh->r_cfg.fc_gateway,
		        nh->r_cfg.fc_ifindex);
	}
}

4288 4289
static int ip6_route_info_append(struct net *net,
				 struct list_head *rt6_nh_list,
4290 4291
				 struct fib6_info *rt,
				 struct fib6_config *r_cfg)
4292 4293 4294 4295 4296
{
	struct rt6_nh *nh;
	int err = -EEXIST;

	list_for_each_entry(nh, rt6_nh_list, next) {
4297 4298
		/* check if fib6_info already exists */
		if (rt6_duplicate_nexthop(nh->fib6_info, rt))
4299 4300 4301 4302 4303 4304
			return err;
	}

	nh = kzalloc(sizeof(*nh), GFP_KERNEL);
	if (!nh)
		return -ENOMEM;
4305
	nh->fib6_info = rt;
4306
	err = ip6_convert_metrics(net, rt, r_cfg);
4307 4308 4309 4310 4311 4312 4313 4314 4315 4316
	if (err) {
		kfree(nh);
		return err;
	}
	memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));
	list_add_tail(&nh->next, rt6_nh_list);

	return 0;
}

4317 4318
static void ip6_route_mpath_notify(struct fib6_info *rt,
				   struct fib6_info *rt_last,
4319 4320 4321 4322 4323 4324 4325 4326 4327
				   struct nl_info *info,
				   __u16 nlflags)
{
	/* if this is an APPEND route, then rt points to the first route
	 * inserted and rt_last points to last route inserted. Userspace
	 * wants a consistent dump of the route which starts at the first
	 * nexthop. Since sibling routes are always added at the end of
	 * the list, find the first sibling of the last route appended
	 */
4328 4329
	if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) {
		rt = list_first_entry(&rt_last->fib6_siblings,
4330
				      struct fib6_info,
4331
				      fib6_siblings);
4332 4333 4334 4335 4336 4337
	}

	if (rt)
		inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
}

4338 4339
static int ip6_route_multipath_add(struct fib6_config *cfg,
				   struct netlink_ext_ack *extack)
4340
{
4341
	struct fib6_info *rt_notif = NULL, *rt_last = NULL;
4342
	struct nl_info *info = &cfg->fc_nlinfo;
4343 4344
	struct fib6_config r_cfg;
	struct rtnexthop *rtnh;
4345
	struct fib6_info *rt;
4346 4347
	struct rt6_nh *err_nh;
	struct rt6_nh *nh, *nh_safe;
4348
	__u16 nlflags;
4349 4350
	int remaining;
	int attrlen;
4351 4352 4353 4354 4355
	int err = 1;
	int nhn = 0;
	int replace = (cfg->fc_nlinfo.nlh &&
		       (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE));
	LIST_HEAD(rt6_nh_list);
4356

4357 4358 4359 4360
	nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE;
	if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND)
		nlflags |= NLM_F_APPEND;

4361
	remaining = cfg->fc_mp_len;
4362 4363
	rtnh = (struct rtnexthop *)cfg->fc_mp;

4364
	/* Parse a Multipath Entry and build a list (rt6_nh_list) of
4365
	 * fib6_info structs per nexthop
4366
	 */
4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377
	while (rtnh_ok(rtnh, remaining)) {
		memcpy(&r_cfg, cfg, sizeof(*cfg));
		if (rtnh->rtnh_ifindex)
			r_cfg.fc_ifindex = rtnh->rtnh_ifindex;

		attrlen = rtnh_attrlen(rtnh);
		if (attrlen > 0) {
			struct nlattr *nla, *attrs = rtnh_attrs(rtnh);

			nla = nla_find(attrs, attrlen, RTA_GATEWAY);
			if (nla) {
4378
				r_cfg.fc_gateway = nla_get_in6_addr(nla);
4379 4380
				r_cfg.fc_flags |= RTF_GATEWAY;
			}
4381 4382 4383 4384
			r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
			nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
			if (nla)
				r_cfg.fc_encap_type = nla_get_u16(nla);
4385
		}
4386

4387
		r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK);
4388
		rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack);
4389 4390 4391
		if (IS_ERR(rt)) {
			err = PTR_ERR(rt);
			rt = NULL;
4392
			goto cleanup;
4393
		}
4394

4395
		rt->fib6_nh.nh_weight = rtnh->rtnh_hops + 1;
4396

4397 4398
		err = ip6_route_info_append(info->nl_net, &rt6_nh_list,
					    rt, &r_cfg);
4399
		if (err) {
4400
			fib6_info_release(rt);
4401 4402 4403 4404 4405 4406
			goto cleanup;
		}

		rtnh = rtnh_next(rtnh, &remaining);
	}

4407 4408 4409 4410 4411 4412
	/* for add and replace send one notification with all nexthops.
	 * Skip the notification in fib6_add_rt2node and send one with
	 * the full route when done
	 */
	info->skip_notify = 1;

4413 4414
	err_nh = NULL;
	list_for_each_entry(nh, &rt6_nh_list, next) {
4415 4416 4417
		rt_last = nh->fib6_info;
		err = __ip6_ins_rt(nh->fib6_info, info, extack);
		fib6_info_release(nh->fib6_info);
4418

4419 4420
		/* save reference to first route for notification */
		if (!rt_notif && !err)
4421
			rt_notif = nh->fib6_info;
4422

4423 4424
		/* nh->fib6_info is used or freed at this point, reset to NULL*/
		nh->fib6_info = NULL;
4425 4426 4427 4428 4429
		if (err) {
			if (replace && nhn)
				ip6_print_replace_route_err(&rt6_nh_list);
			err_nh = nh;
			goto add_errout;
4430
		}
4431

4432
		/* Because each route is added like a single route we remove
4433 4434 4435 4436 4437
		 * these flags after the first nexthop: if there is a collision,
		 * we have already failed to add the first nexthop:
		 * fib6_add_rt2node() has rejected it; when replacing, old
		 * nexthops have been replaced by first new, the rest should
		 * be added to it.
4438
		 */
4439 4440
		cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL |
						     NLM_F_REPLACE);
4441
		cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_APPEND;
4442 4443 4444
		nhn++;
	}

4445 4446
	/* success ... tell user about new route */
	ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
4447 4448 4449
	goto cleanup;

add_errout:
4450 4451 4452 4453 4454 4455 4456
	/* send notification for routes that were added so that
	 * the delete notifications sent by ip6_route_del are
	 * coherent
	 */
	if (rt_notif)
		ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);

4457 4458 4459 4460
	/* Delete routes that were already added */
	list_for_each_entry(nh, &rt6_nh_list, next) {
		if (err_nh == nh)
			break;
4461
		ip6_route_del(&nh->r_cfg, extack);
4462 4463 4464 4465
	}

cleanup:
	list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {
4466 4467
		if (nh->fib6_info)
			fib6_info_release(nh->fib6_info);
4468 4469 4470 4471 4472 4473 4474
		list_del(&nh->next);
		kfree(nh);
	}

	return err;
}

4475 4476
static int ip6_route_multipath_del(struct fib6_config *cfg,
				   struct netlink_ext_ack *extack)
4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502
{
	struct fib6_config r_cfg;
	struct rtnexthop *rtnh;
	int remaining;
	int attrlen;
	int err = 1, last_err = 0;

	remaining = cfg->fc_mp_len;
	rtnh = (struct rtnexthop *)cfg->fc_mp;

	/* Parse a Multipath Entry */
	while (rtnh_ok(rtnh, remaining)) {
		memcpy(&r_cfg, cfg, sizeof(*cfg));
		if (rtnh->rtnh_ifindex)
			r_cfg.fc_ifindex = rtnh->rtnh_ifindex;

		attrlen = rtnh_attrlen(rtnh);
		if (attrlen > 0) {
			struct nlattr *nla, *attrs = rtnh_attrs(rtnh);

			nla = nla_find(attrs, attrlen, RTA_GATEWAY);
			if (nla) {
				nla_memcpy(&r_cfg.fc_gateway, nla, 16);
				r_cfg.fc_flags |= RTF_GATEWAY;
			}
		}
4503
		err = ip6_route_del(&r_cfg, extack);
4504 4505 4506
		if (err)
			last_err = err;

4507 4508 4509 4510 4511 4512
		rtnh = rtnh_next(rtnh, &remaining);
	}

	return last_err;
}

4513 4514
static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
			      struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
4515
{
4516 4517
	struct fib6_config cfg;
	int err;
L
Linus Torvalds 已提交
4518

4519
	err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
4520 4521 4522
	if (err < 0)
		return err;

4523
	if (cfg.fc_mp)
4524
		return ip6_route_multipath_del(&cfg, extack);
4525 4526
	else {
		cfg.fc_delete_all_nh = 1;
4527
		return ip6_route_del(&cfg, extack);
4528
	}
L
Linus Torvalds 已提交
4529 4530
}

4531 4532
static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
			      struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
4533
{
4534 4535
	struct fib6_config cfg;
	int err;
L
Linus Torvalds 已提交
4536

4537
	err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
4538 4539 4540
	if (err < 0)
		return err;

4541
	if (cfg.fc_mp)
4542
		return ip6_route_multipath_add(&cfg, extack);
4543
	else
4544
		return ip6_route_add(&cfg, GFP_KERNEL, extack);
L
Linus Torvalds 已提交
4545 4546
}

4547
static size_t rt6_nlmsg_size(struct fib6_info *rt)
4548
{
4549 4550
	int nexthop_len = 0;

4551
	if (rt->fib6_nsiblings) {
4552 4553 4554
		nexthop_len = nla_total_size(0)	 /* RTA_MULTIPATH */
			    + NLA_ALIGN(sizeof(struct rtnexthop))
			    + nla_total_size(16) /* RTA_GATEWAY */
4555
			    + lwtunnel_get_encap_size(rt->fib6_nh.nh_lwtstate);
4556

4557
		nexthop_len *= rt->fib6_nsiblings;
4558 4559
	}

4560 4561 4562 4563 4564 4565 4566 4567 4568
	return NLMSG_ALIGN(sizeof(struct rtmsg))
	       + nla_total_size(16) /* RTA_SRC */
	       + nla_total_size(16) /* RTA_DST */
	       + nla_total_size(16) /* RTA_GATEWAY */
	       + nla_total_size(16) /* RTA_PREFSRC */
	       + nla_total_size(4) /* RTA_TABLE */
	       + nla_total_size(4) /* RTA_IIF */
	       + nla_total_size(4) /* RTA_OIF */
	       + nla_total_size(4) /* RTA_PRIORITY */
4569
	       + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */
4570
	       + nla_total_size(sizeof(struct rta_cacheinfo))
4571
	       + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */
4572
	       + nla_total_size(1) /* RTA_PREF */
4573
	       + lwtunnel_get_encap_size(rt->fib6_nh.nh_lwtstate)
4574 4575 4576
	       + nexthop_len;
}

4577
static int rt6_nexthop_info(struct sk_buff *skb, struct fib6_info *rt,
4578
			    unsigned int *flags, bool skip_oif)
4579
{
4580
	if (rt->fib6_nh.nh_flags & RTNH_F_DEAD)
4581 4582
		*flags |= RTNH_F_DEAD;

4583
	if (rt->fib6_nh.nh_flags & RTNH_F_LINKDOWN) {
4584
		*flags |= RTNH_F_LINKDOWN;
D
David Ahern 已提交
4585 4586 4587

		rcu_read_lock();
		if (fib6_ignore_linkdown(rt))
4588
			*flags |= RTNH_F_DEAD;
D
David Ahern 已提交
4589
		rcu_read_unlock();
4590 4591
	}

4592
	if (rt->fib6_flags & RTF_GATEWAY) {
4593
		if (nla_put_in6_addr(skb, RTA_GATEWAY, &rt->fib6_nh.nh_gw) < 0)
4594 4595 4596
			goto nla_put_failure;
	}

4597 4598
	*flags |= (rt->fib6_nh.nh_flags & RTNH_F_ONLINK);
	if (rt->fib6_nh.nh_flags & RTNH_F_OFFLOAD)
4599 4600
		*flags |= RTNH_F_OFFLOAD;

4601
	/* not needed for multipath encoding b/c it has a rtnexthop struct */
4602 4603
	if (!skip_oif && rt->fib6_nh.nh_dev &&
	    nla_put_u32(skb, RTA_OIF, rt->fib6_nh.nh_dev->ifindex))
4604 4605
		goto nla_put_failure;

4606 4607
	if (rt->fib6_nh.nh_lwtstate &&
	    lwtunnel_fill_encap(skb, rt->fib6_nh.nh_lwtstate) < 0)
4608 4609 4610 4611 4612 4613 4614 4615
		goto nla_put_failure;

	return 0;

nla_put_failure:
	return -EMSGSIZE;
}

4616
/* add multipath next hop */
4617
static int rt6_add_nexthop(struct sk_buff *skb, struct fib6_info *rt)
4618
{
4619
	const struct net_device *dev = rt->fib6_nh.nh_dev;
4620 4621 4622 4623 4624 4625 4626
	struct rtnexthop *rtnh;
	unsigned int flags = 0;

	rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
	if (!rtnh)
		goto nla_put_failure;

4627 4628
	rtnh->rtnh_hops = rt->fib6_nh.nh_weight - 1;
	rtnh->rtnh_ifindex = dev ? dev->ifindex : 0;
4629

4630
	if (rt6_nexthop_info(skb, rt, &flags, true) < 0)
4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641
		goto nla_put_failure;

	rtnh->rtnh_flags = flags;

	/* length of rtnetlink header + attributes */
	rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;

	return 0;

nla_put_failure:
	return -EMSGSIZE;
4642 4643
}

4644
static int rt6_fill_node(struct net *net, struct sk_buff *skb,
4645
			 struct fib6_info *rt, struct dst_entry *dst,
4646
			 struct in6_addr *dest, struct in6_addr *src,
4647
			 int iif, int type, u32 portid, u32 seq,
4648
			 unsigned int flags)
L
Linus Torvalds 已提交
4649 4650
{
	struct rtmsg *rtm;
4651
	struct nlmsghdr *nlh;
4652 4653
	long expires = 0;
	u32 *pmetrics;
4654
	u32 table;
L
Linus Torvalds 已提交
4655

4656
	nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags);
4657
	if (!nlh)
4658
		return -EMSGSIZE;
4659 4660

	rtm = nlmsg_data(nlh);
L
Linus Torvalds 已提交
4661
	rtm->rtm_family = AF_INET6;
4662 4663
	rtm->rtm_dst_len = rt->fib6_dst.plen;
	rtm->rtm_src_len = rt->fib6_src.plen;
L
Linus Torvalds 已提交
4664
	rtm->rtm_tos = 0;
4665 4666
	if (rt->fib6_table)
		table = rt->fib6_table->tb6_id;
T
Thomas Graf 已提交
4667
	else
4668 4669
		table = RT6_TABLE_UNSPEC;
	rtm->rtm_table = table;
D
David S. Miller 已提交
4670 4671
	if (nla_put_u32(skb, RTA_TABLE, table))
		goto nla_put_failure;
4672 4673

	rtm->rtm_type = rt->fib6_type;
L
Linus Torvalds 已提交
4674 4675
	rtm->rtm_flags = 0;
	rtm->rtm_scope = RT_SCOPE_UNIVERSE;
4676
	rtm->rtm_protocol = rt->fib6_protocol;
L
Linus Torvalds 已提交
4677

4678
	if (rt->fib6_flags & RTF_CACHE)
L
Linus Torvalds 已提交
4679 4680
		rtm->rtm_flags |= RTM_F_CLONED;

4681 4682
	if (dest) {
		if (nla_put_in6_addr(skb, RTA_DST, dest))
D
David S. Miller 已提交
4683
			goto nla_put_failure;
4684
		rtm->rtm_dst_len = 128;
L
Linus Torvalds 已提交
4685
	} else if (rtm->rtm_dst_len)
4686
		if (nla_put_in6_addr(skb, RTA_DST, &rt->fib6_dst.addr))
D
David S. Miller 已提交
4687
			goto nla_put_failure;
L
Linus Torvalds 已提交
4688 4689
#ifdef CONFIG_IPV6_SUBTREES
	if (src) {
4690
		if (nla_put_in6_addr(skb, RTA_SRC, src))
D
David S. Miller 已提交
4691
			goto nla_put_failure;
4692
		rtm->rtm_src_len = 128;
D
David S. Miller 已提交
4693
	} else if (rtm->rtm_src_len &&
4694
		   nla_put_in6_addr(skb, RTA_SRC, &rt->fib6_src.addr))
D
David S. Miller 已提交
4695
		goto nla_put_failure;
L
Linus Torvalds 已提交
4696
#endif
4697 4698
	if (iif) {
#ifdef CONFIG_IPV6_MROUTE
4699
		if (ipv6_addr_is_multicast(&rt->fib6_dst.addr)) {
4700 4701 4702 4703 4704 4705
			int err = ip6mr_get_route(net, skb, rtm, portid);

			if (err == 0)
				return 0;
			if (err < 0)
				goto nla_put_failure;
4706 4707
		} else
#endif
D
David S. Miller 已提交
4708 4709
			if (nla_put_u32(skb, RTA_IIF, iif))
				goto nla_put_failure;
4710
	} else if (dest) {
L
Linus Torvalds 已提交
4711
		struct in6_addr saddr_buf;
4712
		if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 &&
4713
		    nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
D
David S. Miller 已提交
4714
			goto nla_put_failure;
L
Linus Torvalds 已提交
4715
	}
4716

4717
	if (rt->fib6_prefsrc.plen) {
4718
		struct in6_addr saddr_buf;
4719
		saddr_buf = rt->fib6_prefsrc.addr;
4720
		if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
D
David S. Miller 已提交
4721
			goto nla_put_failure;
4722 4723
	}

4724 4725
	pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics;
	if (rtnetlink_put_metrics(skb, pmetrics) < 0)
4726 4727
		goto nla_put_failure;

4728
	if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric))
D
David S. Miller 已提交
4729
		goto nla_put_failure;
4730

4731 4732 4733
	/* For multipath routes, walk the siblings list and add
	 * each as a nexthop within RTA_MULTIPATH.
	 */
4734
	if (rt->fib6_nsiblings) {
4735
		struct fib6_info *sibling, *next_sibling;
4736 4737 4738 4739 4740 4741 4742 4743 4744 4745
		struct nlattr *mp;

		mp = nla_nest_start(skb, RTA_MULTIPATH);
		if (!mp)
			goto nla_put_failure;

		if (rt6_add_nexthop(skb, rt) < 0)
			goto nla_put_failure;

		list_for_each_entry_safe(sibling, next_sibling,
4746
					 &rt->fib6_siblings, fib6_siblings) {
4747 4748 4749 4750 4751 4752
			if (rt6_add_nexthop(skb, sibling) < 0)
				goto nla_put_failure;
		}

		nla_nest_end(skb, mp);
	} else {
4753
		if (rt6_nexthop_info(skb, rt, &rtm->rtm_flags, false) < 0)
4754 4755 4756
			goto nla_put_failure;
	}

4757
	if (rt->fib6_flags & RTF_EXPIRES) {
4758 4759 4760
		expires = dst ? dst->expires : rt->expires;
		expires -= jiffies;
	}
4761

4762
	if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0)
4763
		goto nla_put_failure;
4764

4765
	if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt->fib6_flags)))
4766 4767
		goto nla_put_failure;

4768

4769 4770
	nlmsg_end(skb, nlh);
	return 0;
4771 4772

nla_put_failure:
4773 4774
	nlmsg_cancel(skb, nlh);
	return -EMSGSIZE;
L
Linus Torvalds 已提交
4775 4776
}

4777
int rt6_dump_route(struct fib6_info *rt, void *p_arg)
L
Linus Torvalds 已提交
4778 4779
{
	struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg;
4780 4781
	struct net *net = arg->net;

D
David Ahern 已提交
4782
	if (rt == net->ipv6.fib6_null_entry)
4783
		return 0;
L
Linus Torvalds 已提交
4784

4785 4786
	if (nlmsg_len(arg->cb->nlh) >= sizeof(struct rtmsg)) {
		struct rtmsg *rtm = nlmsg_data(arg->cb->nlh);
4787 4788 4789

		/* user wants prefix routes only */
		if (rtm->rtm_flags & RTM_F_PREFIX &&
4790
		    !(rt->fib6_flags & RTF_PREFIX_RT)) {
4791 4792 4793 4794
			/* success since this is not a prefix route */
			return 1;
		}
	}
L
Linus Torvalds 已提交
4795

4796 4797 4798
	return rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL, 0,
			     RTM_NEWROUTE, NETLINK_CB(arg->cb->skb).portid,
			     arg->cb->nlh->nlmsg_seq, NLM_F_MULTI);
L
Linus Torvalds 已提交
4799 4800
}

4801 4802
static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
			      struct netlink_ext_ack *extack)
L
Linus Torvalds 已提交
4803
{
4804
	struct net *net = sock_net(in_skb->sk);
4805
	struct nlattr *tb[RTA_MAX+1];
4806
	int err, iif = 0, oif = 0;
4807
	struct fib6_info *from;
4808
	struct dst_entry *dst;
4809
	struct rt6_info *rt;
L
Linus Torvalds 已提交
4810
	struct sk_buff *skb;
4811
	struct rtmsg *rtm;
4812
	struct flowi6 fl6;
4813
	bool fibmatch;
L
Linus Torvalds 已提交
4814

4815
	err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv6_policy,
4816
			  extack);
4817 4818
	if (err < 0)
		goto errout;
L
Linus Torvalds 已提交
4819

4820
	err = -EINVAL;
4821
	memset(&fl6, 0, sizeof(fl6));
4822 4823
	rtm = nlmsg_data(nlh);
	fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0);
4824
	fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH);
L
Linus Torvalds 已提交
4825

4826 4827 4828 4829
	if (tb[RTA_SRC]) {
		if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr))
			goto errout;

A
Alexey Dobriyan 已提交
4830
		fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]);
4831 4832 4833 4834 4835 4836
	}

	if (tb[RTA_DST]) {
		if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr))
			goto errout;

A
Alexey Dobriyan 已提交
4837
		fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]);
4838 4839 4840 4841 4842 4843
	}

	if (tb[RTA_IIF])
		iif = nla_get_u32(tb[RTA_IIF]);

	if (tb[RTA_OIF])
4844
		oif = nla_get_u32(tb[RTA_OIF]);
L
Linus Torvalds 已提交
4845

4846 4847 4848
	if (tb[RTA_MARK])
		fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]);

4849 4850 4851 4852 4853 4854
	if (tb[RTA_UID])
		fl6.flowi6_uid = make_kuid(current_user_ns(),
					   nla_get_u32(tb[RTA_UID]));
	else
		fl6.flowi6_uid = iif ? INVALID_UID : current_uid();

4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867
	if (tb[RTA_SPORT])
		fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]);

	if (tb[RTA_DPORT])
		fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]);

	if (tb[RTA_IP_PROTO]) {
		err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
						  &fl6.flowi6_proto, extack);
		if (err)
			goto errout;
	}

L
Linus Torvalds 已提交
4868 4869
	if (iif) {
		struct net_device *dev;
4870 4871
		int flags = 0;

4872 4873 4874
		rcu_read_lock();

		dev = dev_get_by_index_rcu(net, iif);
L
Linus Torvalds 已提交
4875
		if (!dev) {
4876
			rcu_read_unlock();
L
Linus Torvalds 已提交
4877
			err = -ENODEV;
4878
			goto errout;
L
Linus Torvalds 已提交
4879
		}
4880 4881 4882 4883 4884 4885

		fl6.flowi6_iif = iif;

		if (!ipv6_addr_any(&fl6.saddr))
			flags |= RT6_LOOKUP_F_HAS_SADDR;

D
David Ahern 已提交
4886
		dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags);
4887 4888

		rcu_read_unlock();
4889 4890 4891
	} else {
		fl6.flowi6_oif = oif;

4892
		dst = ip6_route_output(net, NULL, &fl6);
4893 4894 4895 4896 4897 4898 4899 4900
	}


	rt = container_of(dst, struct rt6_info, dst);
	if (rt->dst.error) {
		err = rt->dst.error;
		ip6_rt_put(rt);
		goto errout;
L
Linus Torvalds 已提交
4901 4902
	}

4903 4904 4905 4906 4907 4908
	if (rt == net->ipv6.ip6_null_entry) {
		err = rt->dst.error;
		ip6_rt_put(rt);
		goto errout;
	}

4909
	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
4910
	if (!skb) {
A
Amerigo Wang 已提交
4911
		ip6_rt_put(rt);
4912 4913 4914
		err = -ENOBUFS;
		goto errout;
	}
L
Linus Torvalds 已提交
4915

4916
	skb_dst_set(skb, &rt->dst);
4917 4918 4919 4920

	rcu_read_lock();
	from = rcu_dereference(rt->from);

4921
	if (fibmatch)
4922
		err = rt6_fill_node(net, skb, from, NULL, NULL, NULL, iif,
4923 4924 4925
				    RTM_NEWROUTE, NETLINK_CB(in_skb).portid,
				    nlh->nlmsg_seq, 0);
	else
4926 4927
		err = rt6_fill_node(net, skb, from, dst, &fl6.daddr,
				    &fl6.saddr, iif, RTM_NEWROUTE,
4928 4929
				    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
				    0);
4930 4931
	rcu_read_unlock();

L
Linus Torvalds 已提交
4932
	if (err < 0) {
4933 4934
		kfree_skb(skb);
		goto errout;
L
Linus Torvalds 已提交
4935 4936
	}

4937
	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
4938
errout:
L
Linus Torvalds 已提交
4939 4940 4941
	return err;
}

4942
void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
4943
		     unsigned int nlm_flags)
L
Linus Torvalds 已提交
4944 4945
{
	struct sk_buff *skb;
4946
	struct net *net = info->nl_net;
4947 4948 4949 4950
	u32 seq;
	int err;

	err = -ENOBUFS;
4951
	seq = info->nlh ? info->nlh->nlmsg_seq : 0;
4952

4953
	skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
4954
	if (!skb)
4955 4956
		goto errout;

4957 4958
	err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
			    event, info->portid, seq, nlm_flags);
4959 4960 4961 4962 4963 4964
	if (err < 0) {
		/* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
		WARN_ON(err == -EMSGSIZE);
		kfree_skb(skb);
		goto errout;
	}
4965
	rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
4966 4967
		    info->nlh, gfp_any());
	return;
4968 4969
errout:
	if (err < 0)
4970
		rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
L
Linus Torvalds 已提交
4971 4972
}

4973
static int ip6_route_dev_notify(struct notifier_block *this,
4974
				unsigned long event, void *ptr)
4975
{
4976
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4977
	struct net *net = dev_net(dev);
4978

4979 4980 4981 4982
	if (!(dev->flags & IFF_LOOPBACK))
		return NOTIFY_OK;

	if (event == NETDEV_REGISTER) {
D
David Ahern 已提交
4983
		net->ipv6.fib6_null_entry->fib6_nh.nh_dev = dev;
4984
		net->ipv6.ip6_null_entry->dst.dev = dev;
4985 4986
		net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev);
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
4987
		net->ipv6.ip6_prohibit_entry->dst.dev = dev;
4988
		net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev);
4989
		net->ipv6.ip6_blk_hole_entry->dst.dev = dev;
4990
		net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev);
4991
#endif
4992 4993 4994 4995 4996
	 } else if (event == NETDEV_UNREGISTER &&
		    dev->reg_state != NETREG_UNREGISTERED) {
		/* NETDEV_UNREGISTER could be fired for multiple times by
		 * netdev_wait_allrefs(). Make sure we only call this once.
		 */
4997
		in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev);
4998
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
4999 5000
		in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev);
		in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev);
5001 5002 5003 5004 5005 5006
#endif
	}

	return NOTIFY_OK;
}

L
Linus Torvalds 已提交
5007 5008 5009 5010 5011 5012
/*
 *	/proc
 */

#ifdef CONFIG_PROC_FS

5013 5014 5015 5016
static const struct file_operations ipv6_route_proc_fops = {
	.open		= ipv6_route_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
5017
	.release	= seq_release_net,
5018 5019
};

L
Linus Torvalds 已提交
5020 5021
static int rt6_stats_seq_show(struct seq_file *seq, void *v)
{
5022
	struct net *net = (struct net *)seq->private;
L
Linus Torvalds 已提交
5023
	seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n",
5024 5025
		   net->ipv6.rt6_stats->fib_nodes,
		   net->ipv6.rt6_stats->fib_route_nodes,
W
Wei Wang 已提交
5026
		   atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc),
5027 5028
		   net->ipv6.rt6_stats->fib_rt_entries,
		   net->ipv6.rt6_stats->fib_rt_cache,
5029
		   dst_entries_get_slow(&net->ipv6.ip6_dst_ops),
5030
		   net->ipv6.rt6_stats->fib_discarded_routes);
L
Linus Torvalds 已提交
5031 5032 5033 5034 5035 5036

	return 0;
}

static int rt6_stats_seq_open(struct inode *inode, struct file *file)
{
5037
	return single_open_net(inode, file, rt6_stats_seq_show);
5038 5039
}

5040
static const struct file_operations rt6_stats_seq_fops = {
L
Linus Torvalds 已提交
5041 5042 5043
	.open	 = rt6_stats_seq_open,
	.read	 = seq_read,
	.llseek	 = seq_lseek,
5044
	.release = single_release_net,
L
Linus Torvalds 已提交
5045 5046 5047 5048 5049 5050
};
#endif	/* CONFIG_PROC_FS */

#ifdef CONFIG_SYSCTL

static
5051
int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write,
L
Linus Torvalds 已提交
5052 5053
			      void __user *buffer, size_t *lenp, loff_t *ppos)
{
5054 5055 5056
	struct net *net;
	int delay;
	if (!write)
L
Linus Torvalds 已提交
5057
		return -EINVAL;
5058 5059 5060 5061

	net = (struct net *)ctl->extra1;
	delay = net->ipv6.sysctl.flush_delay;
	proc_dointvec(ctl, write, buffer, lenp, ppos);
5062
	fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0);
5063
	return 0;
L
Linus Torvalds 已提交
5064 5065
}

5066
struct ctl_table ipv6_route_table_template[] = {
5067
	{
L
Linus Torvalds 已提交
5068
		.procname	=	"flush",
5069
		.data		=	&init_net.ipv6.sysctl.flush_delay,
L
Linus Torvalds 已提交
5070
		.maxlen		=	sizeof(int),
5071
		.mode		=	0200,
A
Alexey Dobriyan 已提交
5072
		.proc_handler	=	ipv6_sysctl_rtcache_flush
L
Linus Torvalds 已提交
5073 5074 5075
	},
	{
		.procname	=	"gc_thresh",
5076
		.data		=	&ip6_dst_ops_template.gc_thresh,
L
Linus Torvalds 已提交
5077 5078
		.maxlen		=	sizeof(int),
		.mode		=	0644,
A
Alexey Dobriyan 已提交
5079
		.proc_handler	=	proc_dointvec,
L
Linus Torvalds 已提交
5080 5081 5082
	},
	{
		.procname	=	"max_size",
5083
		.data		=	&init_net.ipv6.sysctl.ip6_rt_max_size,
L
Linus Torvalds 已提交
5084 5085
		.maxlen		=	sizeof(int),
		.mode		=	0644,
A
Alexey Dobriyan 已提交
5086
		.proc_handler	=	proc_dointvec,
L
Linus Torvalds 已提交
5087 5088 5089
	},
	{
		.procname	=	"gc_min_interval",
5090
		.data		=	&init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
L
Linus Torvalds 已提交
5091 5092
		.maxlen		=	sizeof(int),
		.mode		=	0644,
A
Alexey Dobriyan 已提交
5093
		.proc_handler	=	proc_dointvec_jiffies,
L
Linus Torvalds 已提交
5094 5095 5096
	},
	{
		.procname	=	"gc_timeout",
5097
		.data		=	&init_net.ipv6.sysctl.ip6_rt_gc_timeout,
L
Linus Torvalds 已提交
5098 5099
		.maxlen		=	sizeof(int),
		.mode		=	0644,
A
Alexey Dobriyan 已提交
5100
		.proc_handler	=	proc_dointvec_jiffies,
L
Linus Torvalds 已提交
5101 5102 5103
	},
	{
		.procname	=	"gc_interval",
5104
		.data		=	&init_net.ipv6.sysctl.ip6_rt_gc_interval,
L
Linus Torvalds 已提交
5105 5106
		.maxlen		=	sizeof(int),
		.mode		=	0644,
A
Alexey Dobriyan 已提交
5107
		.proc_handler	=	proc_dointvec_jiffies,
L
Linus Torvalds 已提交
5108 5109 5110
	},
	{
		.procname	=	"gc_elasticity",
5111
		.data		=	&init_net.ipv6.sysctl.ip6_rt_gc_elasticity,
L
Linus Torvalds 已提交
5112 5113
		.maxlen		=	sizeof(int),
		.mode		=	0644,
5114
		.proc_handler	=	proc_dointvec,
L
Linus Torvalds 已提交
5115 5116 5117
	},
	{
		.procname	=	"mtu_expires",
5118
		.data		=	&init_net.ipv6.sysctl.ip6_rt_mtu_expires,
L
Linus Torvalds 已提交
5119 5120
		.maxlen		=	sizeof(int),
		.mode		=	0644,
A
Alexey Dobriyan 已提交
5121
		.proc_handler	=	proc_dointvec_jiffies,
L
Linus Torvalds 已提交
5122 5123 5124
	},
	{
		.procname	=	"min_adv_mss",
5125
		.data		=	&init_net.ipv6.sysctl.ip6_rt_min_advmss,
L
Linus Torvalds 已提交
5126 5127
		.maxlen		=	sizeof(int),
		.mode		=	0644,
5128
		.proc_handler	=	proc_dointvec,
L
Linus Torvalds 已提交
5129 5130 5131
	},
	{
		.procname	=	"gc_min_interval_ms",
5132
		.data		=	&init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
L
Linus Torvalds 已提交
5133 5134
		.maxlen		=	sizeof(int),
		.mode		=	0644,
A
Alexey Dobriyan 已提交
5135
		.proc_handler	=	proc_dointvec_ms_jiffies,
L
Linus Torvalds 已提交
5136
	},
5137
	{ }
L
Linus Torvalds 已提交
5138 5139
};

5140
struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net)
5141 5142 5143 5144 5145 5146
{
	struct ctl_table *table;

	table = kmemdup(ipv6_route_table_template,
			sizeof(ipv6_route_table_template),
			GFP_KERNEL);
5147 5148 5149

	if (table) {
		table[0].data = &net->ipv6.sysctl.flush_delay;
5150
		table[0].extra1 = net;
5151
		table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh;
5152 5153 5154 5155 5156 5157 5158
		table[2].data = &net->ipv6.sysctl.ip6_rt_max_size;
		table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
		table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout;
		table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval;
		table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity;
		table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires;
		table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss;
5159
		table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
5160 5161 5162 5163

		/* Don't export sysctls to unprivileged users */
		if (net->user_ns != &init_user_ns)
			table[0].procname = NULL;
5164 5165
	}

5166 5167
	return table;
}
L
Linus Torvalds 已提交
5168 5169
#endif

5170
static int __net_init ip6_route_net_init(struct net *net)
5171
{
5172
	int ret = -ENOMEM;
5173

5174 5175
	memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template,
	       sizeof(net->ipv6.ip6_dst_ops));
5176

5177 5178 5179
	if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0)
		goto out_ip6_dst_ops;

D
David Ahern 已提交
5180 5181 5182 5183 5184 5185
	net->ipv6.fib6_null_entry = kmemdup(&fib6_null_entry_template,
					    sizeof(*net->ipv6.fib6_null_entry),
					    GFP_KERNEL);
	if (!net->ipv6.fib6_null_entry)
		goto out_ip6_dst_entries;

5186 5187 5188 5189
	net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template,
					   sizeof(*net->ipv6.ip6_null_entry),
					   GFP_KERNEL);
	if (!net->ipv6.ip6_null_entry)
D
David Ahern 已提交
5190
		goto out_fib6_null_entry;
5191
	net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops;
5192 5193
	dst_init_metrics(&net->ipv6.ip6_null_entry->dst,
			 ip6_template_metrics, true);
5194 5195

#ifdef CONFIG_IPV6_MULTIPLE_TABLES
5196
	net->ipv6.fib6_has_custom_rules = false;
5197 5198 5199
	net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template,
					       sizeof(*net->ipv6.ip6_prohibit_entry),
					       GFP_KERNEL);
5200 5201
	if (!net->ipv6.ip6_prohibit_entry)
		goto out_ip6_null_entry;
5202
	net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops;
5203 5204
	dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst,
			 ip6_template_metrics, true);
5205 5206 5207 5208

	net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template,
					       sizeof(*net->ipv6.ip6_blk_hole_entry),
					       GFP_KERNEL);
5209 5210
	if (!net->ipv6.ip6_blk_hole_entry)
		goto out_ip6_prohibit_entry;
5211
	net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops;
5212 5213
	dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst,
			 ip6_template_metrics, true);
5214 5215
#endif

5216 5217 5218 5219 5220 5221 5222 5223 5224
	net->ipv6.sysctl.flush_delay = 0;
	net->ipv6.sysctl.ip6_rt_max_size = 4096;
	net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
	net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
	net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
	net->ipv6.sysctl.ip6_rt_gc_elasticity = 9;
	net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ;
	net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40;

5225 5226
	net->ipv6.ip6_rt_gc_expire = 30*HZ;

5227 5228 5229
	ret = 0;
out:
	return ret;
5230

5231 5232 5233 5234 5235 5236
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
out_ip6_prohibit_entry:
	kfree(net->ipv6.ip6_prohibit_entry);
out_ip6_null_entry:
	kfree(net->ipv6.ip6_null_entry);
#endif
D
David Ahern 已提交
5237 5238
out_fib6_null_entry:
	kfree(net->ipv6.fib6_null_entry);
5239 5240
out_ip6_dst_entries:
	dst_entries_destroy(&net->ipv6.ip6_dst_ops);
5241 5242
out_ip6_dst_ops:
	goto out;
5243 5244
}

5245
static void __net_exit ip6_route_net_exit(struct net *net)
5246
{
D
David Ahern 已提交
5247
	kfree(net->ipv6.fib6_null_entry);
5248 5249 5250 5251 5252
	kfree(net->ipv6.ip6_null_entry);
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
	kfree(net->ipv6.ip6_prohibit_entry);
	kfree(net->ipv6.ip6_blk_hole_entry);
#endif
5253
	dst_entries_destroy(&net->ipv6.ip6_dst_ops);
5254 5255
}

5256 5257 5258
static int __net_init ip6_route_net_init_late(struct net *net)
{
#ifdef CONFIG_PROC_FS
5259
	proc_create("ipv6_route", 0, net->proc_net, &ipv6_route_proc_fops);
5260
	proc_create("rt6_stats", 0444, net->proc_net, &rt6_stats_seq_fops);
5261 5262 5263 5264 5265 5266 5267
#endif
	return 0;
}

static void __net_exit ip6_route_net_exit_late(struct net *net)
{
#ifdef CONFIG_PROC_FS
5268 5269
	remove_proc_entry("ipv6_route", net->proc_net);
	remove_proc_entry("rt6_stats", net->proc_net);
5270 5271 5272
#endif
}

5273 5274 5275 5276 5277
static struct pernet_operations ip6_route_net_ops = {
	.init = ip6_route_net_init,
	.exit = ip6_route_net_exit,
};

5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293
static int __net_init ipv6_inetpeer_init(struct net *net)
{
	struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);

	if (!bp)
		return -ENOMEM;
	inet_peer_base_init(bp);
	net->ipv6.peers = bp;
	return 0;
}

static void __net_exit ipv6_inetpeer_exit(struct net *net)
{
	struct inet_peer_base *bp = net->ipv6.peers;

	net->ipv6.peers = NULL;
5294
	inetpeer_invalidate_tree(bp);
5295 5296 5297
	kfree(bp);
}

5298
static struct pernet_operations ipv6_inetpeer_ops = {
5299 5300 5301 5302
	.init	=	ipv6_inetpeer_init,
	.exit	=	ipv6_inetpeer_exit,
};

5303 5304 5305 5306 5307
static struct pernet_operations ip6_route_net_late_ops = {
	.init = ip6_route_net_init_late,
	.exit = ip6_route_net_exit_late,
};

5308 5309
static struct notifier_block ip6_route_dev_notifier = {
	.notifier_call = ip6_route_dev_notify,
5310
	.priority = ADDRCONF_NOTIFY_PRIORITY - 10,
5311 5312
};

5313 5314 5315 5316 5317
void __init ip6_route_init_special_entries(void)
{
	/* Registering of the loopback is done before this portion of code,
	 * the loopback reference in rt6_info will not be taken, do it
	 * manually for init_net */
D
David Ahern 已提交
5318
	init_net.ipv6.fib6_null_entry->fib6_nh.nh_dev = init_net.loopback_dev;
5319 5320 5321 5322 5323 5324 5325 5326 5327 5328
	init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev;
	init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
  #ifdef CONFIG_IPV6_MULTIPLE_TABLES
	init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev;
	init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
	init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev;
	init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
  #endif
}

5329
int __init ip6_route_init(void)
L
Linus Torvalds 已提交
5330
{
5331
	int ret;
5332
	int cpu;
5333

5334 5335
	ret = -ENOMEM;
	ip6_dst_ops_template.kmem_cachep =
A
Alexey Dobriyan 已提交
5336
		kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0,
5337
				  SLAB_HWCACHE_ALIGN, NULL);
5338
	if (!ip6_dst_ops_template.kmem_cachep)
5339
		goto out;
5340

5341
	ret = dst_entries_init(&ip6_dst_blackhole_ops);
5342
	if (ret)
5343 5344
		goto out_kmem_cache;

5345 5346
	ret = register_pernet_subsys(&ipv6_inetpeer_ops);
	if (ret)
5347
		goto out_dst_entries;
5348

5349 5350 5351
	ret = register_pernet_subsys(&ip6_route_net_ops);
	if (ret)
		goto out_register_inetpeer;
5352

5353 5354
	ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep;

5355
	ret = fib6_init();
5356
	if (ret)
5357
		goto out_register_subsys;
5358 5359 5360

	ret = xfrm6_init();
	if (ret)
5361
		goto out_fib6_init;
5362

5363 5364 5365
	ret = fib6_rules_init();
	if (ret)
		goto xfrm6_init;
5366

5367 5368 5369 5370
	ret = register_pernet_subsys(&ip6_route_net_late_ops);
	if (ret)
		goto fib6_rules_init;

5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384
	ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE,
				   inet6_rtm_newroute, NULL, 0);
	if (ret < 0)
		goto out_register_late_subsys;

	ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE,
				   inet6_rtm_delroute, NULL, 0);
	if (ret < 0)
		goto out_register_late_subsys;

	ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE,
				   inet6_rtm_getroute, NULL,
				   RTNL_FLAG_DOIT_UNLOCKED);
	if (ret < 0)
5385
		goto out_register_late_subsys;
5386

5387
	ret = register_netdevice_notifier(&ip6_route_dev_notifier);
5388
	if (ret)
5389
		goto out_register_late_subsys;
5390

5391 5392 5393 5394 5395 5396 5397
	for_each_possible_cpu(cpu) {
		struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);

		INIT_LIST_HEAD(&ul->head);
		spin_lock_init(&ul->lock);
	}

5398 5399 5400
out:
	return ret;

5401
out_register_late_subsys:
5402
	rtnl_unregister_all(PF_INET6);
5403
	unregister_pernet_subsys(&ip6_route_net_late_ops);
5404 5405 5406 5407
fib6_rules_init:
	fib6_rules_cleanup();
xfrm6_init:
	xfrm6_fini();
5408 5409
out_fib6_init:
	fib6_gc_cleanup();
5410 5411
out_register_subsys:
	unregister_pernet_subsys(&ip6_route_net_ops);
5412 5413
out_register_inetpeer:
	unregister_pernet_subsys(&ipv6_inetpeer_ops);
5414 5415
out_dst_entries:
	dst_entries_destroy(&ip6_dst_blackhole_ops);
5416
out_kmem_cache:
5417
	kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
5418
	goto out;
L
Linus Torvalds 已提交
5419 5420 5421 5422
}

void ip6_route_cleanup(void)
{
5423
	unregister_netdevice_notifier(&ip6_route_dev_notifier);
5424
	unregister_pernet_subsys(&ip6_route_net_late_ops);
T
Thomas Graf 已提交
5425
	fib6_rules_cleanup();
L
Linus Torvalds 已提交
5426 5427
	xfrm6_fini();
	fib6_gc_cleanup();
5428
	unregister_pernet_subsys(&ipv6_inetpeer_ops);
5429
	unregister_pernet_subsys(&ip6_route_net_ops);
5430
	dst_entries_destroy(&ip6_dst_blackhole_ops);
5431
	kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
L
Linus Torvalds 已提交
5432
}