netpoll.c 30.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 * Common framework for low-level network console, dump, and debugger code
 *
 * Sep 8 2003  Matt Mackall <mpm@selenic.com>
 *
 * based on the netconsole code from:
 *
 * Copyright (C) 2001  Ingo Molnar <mingo@redhat.com>
 * Copyright (C) 2002  Red Hat, Inc.
 */

12 13
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14
#include <linux/moduleparam.h>
15
#include <linux/kernel.h>
L
Linus Torvalds 已提交
16 17 18
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/string.h>
19
#include <linux/if_arp.h>
L
Linus Torvalds 已提交
20 21 22 23 24 25 26 27
#include <linux/inetdevice.h>
#include <linux/inet.h>
#include <linux/interrupt.h>
#include <linux/netpoll.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
28
#include <linux/slab.h>
29
#include <linux/export.h>
30
#include <linux/if_vlan.h>
L
Linus Torvalds 已提交
31 32
#include <net/tcp.h>
#include <net/udp.h>
C
Cong Wang 已提交
33 34 35
#include <net/addrconf.h>
#include <net/ndisc.h>
#include <net/ip6_checksum.h>
L
Linus Torvalds 已提交
36
#include <asm/unaligned.h>
37
#include <trace/events/napi.h>
L
Linus Torvalds 已提交
38 39 40 41 42 43 44 45 46

/*
 * We maintain a small pool of fully-sized skbs, to make sure the
 * message gets out even in extreme OOM situations.
 */

#define MAX_UDP_CHUNK 1460
#define MAX_SKBS 32

47
static struct sk_buff_head skb_pool;
L
Linus Torvalds 已提交
48 49 50

static atomic_t trapped;

51
DEFINE_STATIC_SRCU(netpoll_srcu);
52

S
Stephen Hemminger 已提交
53
#define USEC_PER_POLL	50
54 55
#define NETPOLL_RX_ENABLED  1
#define NETPOLL_RX_DROP     2
L
Linus Torvalds 已提交
56

J
Joe Perches 已提交
57 58 59 60 61
#define MAX_SKB_SIZE							\
	(sizeof(struct ethhdr) +					\
	 sizeof(struct iphdr) +						\
	 sizeof(struct udphdr) +					\
	 MAX_UDP_CHUNK)
L
Linus Torvalds 已提交
62

63
static void zap_completion_queue(void);
C
Cong Wang 已提交
64
static void netpoll_neigh_reply(struct sk_buff *skb, struct netpoll_info *npinfo);
65
static void netpoll_async_cleanup(struct work_struct *work);
L
Linus Torvalds 已提交
66

67 68 69
static unsigned int carrier_timeout = 4;
module_param(carrier_timeout, uint, 0644);

70 71 72 73 74 75 76
#define np_info(np, fmt, ...)				\
	pr_info("%s: " fmt, np->name, ##__VA_ARGS__)
#define np_err(np, fmt, ...)				\
	pr_err("%s: " fmt, np->name, ##__VA_ARGS__)
#define np_notice(np, fmt, ...)				\
	pr_notice("%s: " fmt, np->name, ##__VA_ARGS__)

D
David Howells 已提交
77
static void queue_process(struct work_struct *work)
L
Linus Torvalds 已提交
78
{
79 80
	struct netpoll_info *npinfo =
		container_of(work, struct netpoll_info, tx_work.work);
L
Linus Torvalds 已提交
81
	struct sk_buff *skb;
I
Ingo Molnar 已提交
82
	unsigned long flags;
L
Linus Torvalds 已提交
83

S
Stephen Hemminger 已提交
84 85
	while ((skb = skb_dequeue(&npinfo->txq))) {
		struct net_device *dev = skb->dev;
86
		const struct net_device_ops *ops = dev->netdev_ops;
87
		struct netdev_queue *txq;
L
Linus Torvalds 已提交
88

S
Stephen Hemminger 已提交
89 90 91 92
		if (!netif_device_present(dev) || !netif_running(dev)) {
			__kfree_skb(skb);
			continue;
		}
L
Linus Torvalds 已提交
93

94 95
		txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));

I
Ingo Molnar 已提交
96
		local_irq_save(flags);
97
		__netif_tx_lock(txq, smp_processor_id());
98
		if (netif_xmit_frozen_or_stopped(txq) ||
99
		    ops->ndo_start_xmit(skb, dev) != NETDEV_TX_OK) {
S
Stephen Hemminger 已提交
100
			skb_queue_head(&npinfo->txq, skb);
101
			__netif_tx_unlock(txq);
I
Ingo Molnar 已提交
102
			local_irq_restore(flags);
L
Linus Torvalds 已提交
103

104
			schedule_delayed_work(&npinfo->tx_work, HZ/10);
S
Stephen Hemminger 已提交
105 106
			return;
		}
107
		__netif_tx_unlock(txq);
I
Ingo Molnar 已提交
108
		local_irq_restore(flags);
L
Linus Torvalds 已提交
109 110 111
	}
}

112 113
static __sum16 checksum_udp(struct sk_buff *skb, struct udphdr *uh,
			    unsigned short ulen, __be32 saddr, __be32 daddr)
L
Linus Torvalds 已提交
114
{
115
	__wsum psum;
116

117
	if (uh->check == 0 || skb_csum_unnecessary(skb))
L
Linus Torvalds 已提交
118 119
		return 0;

120 121
	psum = csum_tcpudp_nofold(saddr, daddr, ulen, IPPROTO_UDP, 0);

122
	if (skb->ip_summed == CHECKSUM_COMPLETE &&
123
	    !csum_fold(csum_add(psum, skb->csum)))
124
		return 0;
L
Linus Torvalds 已提交
125

126
	skb->csum = psum;
L
Linus Torvalds 已提交
127

128
	return __skb_checksum_complete(skb);
L
Linus Torvalds 已提交
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
}

/*
 * Check whether delayed processing was scheduled for our NIC. If so,
 * we attempt to grab the poll lock and use ->poll() to pump the card.
 * If this fails, either we've recursed in ->poll() or it's already
 * running on another CPU.
 *
 * Note: we don't mask interrupts with this lock because we're using
 * trylock here and interrupts are already disabled in the softirq
 * case. Further, we test the poll_owner to avoid recursion on UP
 * systems where the lock doesn't exist.
 *
 * In cases where there is bi-directional communications, reading only
 * one message at a time can lead to packets being dropped by the
 * network adapter, forcing superfluous retries and possibly timeouts.
 * Thus, we set our budget to greater than 1.
 */
147 148 149 150 151 152 153 154 155 156 157 158
static int poll_one_napi(struct netpoll_info *npinfo,
			 struct napi_struct *napi, int budget)
{
	int work;

	/* net_rx_action's ->poll() invocations and our's are
	 * synchronized by this test which is only made while
	 * holding the napi->poll_lock.
	 */
	if (!test_bit(NAPI_STATE_SCHED, &napi->state))
		return budget;

159
	npinfo->rx_flags |= NETPOLL_RX_DROP;
160
	atomic_inc(&trapped);
161
	set_bit(NAPI_STATE_NPSVC, &napi->state);
162 163

	work = napi->poll(napi, budget);
164
	trace_napi_poll(napi);
165

166
	clear_bit(NAPI_STATE_NPSVC, &napi->state);
167
	atomic_dec(&trapped);
168
	npinfo->rx_flags &= ~NETPOLL_RX_DROP;
169 170 171 172

	return budget - work;
}

173
static void poll_napi(struct net_device *dev)
L
Linus Torvalds 已提交
174
{
175
	struct napi_struct *napi;
L
Linus Torvalds 已提交
176 177
	int budget = 16;

178
	list_for_each_entry(napi, &dev->napi_list, dev_list) {
179
		if (napi->poll_owner != smp_processor_id() &&
180
		    spin_trylock(&napi->poll_lock)) {
181 182
			budget = poll_one_napi(rcu_dereference_bh(dev->npinfo),
					       napi, budget);
183
			spin_unlock(&napi->poll_lock);
184

185
			if (!budget)
186
				break;
187
		}
L
Linus Torvalds 已提交
188 189 190
	}
}

C
Cong Wang 已提交
191
static void service_neigh_queue(struct netpoll_info *npi)
192
{
193 194
	if (npi) {
		struct sk_buff *skb;
195

C
Cong Wang 已提交
196 197
		while ((skb = skb_dequeue(&npi->neigh_tx)))
			netpoll_neigh_reply(skb, npi);
198 199 200
	}
}

201
static void netpoll_poll_dev(struct net_device *dev)
L
Linus Torvalds 已提交
202
{
203
	const struct net_device_ops *ops;
204
	struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
205

206 207 208 209
	/* Don't do any rx activity if the dev_lock mutex is held
	 * the dev_open/close paths use this to block netpoll activity
	 * while changing device state
	 */
210
	if (down_trylock(&ni->dev_lock))
211 212
		return;

213
	if (!netif_running(dev)) {
214
		up(&ni->dev_lock);
215
		return;
216
	}
217 218

	ops = dev->netdev_ops;
219
	if (!ops->ndo_poll_controller) {
220
		up(&ni->dev_lock);
L
Linus Torvalds 已提交
221
		return;
222
	}
L
Linus Torvalds 已提交
223 224

	/* Process pending work on NIC */
225
	ops->ndo_poll_controller(dev);
226 227

	poll_napi(dev);
L
Linus Torvalds 已提交
228

229
	up(&ni->dev_lock);
230

231
	if (dev->flags & IFF_SLAVE) {
232
		if (ni) {
233
			struct net_device *bond_dev;
234
			struct sk_buff *skb;
235 236 237 238
			struct netpoll_info *bond_ni;

			bond_dev = netdev_master_upper_dev_get_rcu(dev);
			bond_ni = rcu_dereference_bh(bond_dev->npinfo);
C
Cong Wang 已提交
239
			while ((skb = skb_dequeue(&ni->neigh_tx))) {
240
				skb->dev = bond_dev;
C
Cong Wang 已提交
241
				skb_queue_tail(&bond_ni->neigh_tx, skb);
242 243 244 245
			}
		}
	}

C
Cong Wang 已提交
246
	service_neigh_queue(ni);
247

248
	zap_completion_queue();
L
Linus Torvalds 已提交
249 250
}

251
void netpoll_rx_disable(struct net_device *dev)
252 253 254 255 256 257 258
{
	struct netpoll_info *ni;
	int idx;
	might_sleep();
	idx = srcu_read_lock(&netpoll_srcu);
	ni = srcu_dereference(dev->npinfo, &netpoll_srcu);
	if (ni)
259
		down(&ni->dev_lock);
260 261 262 263 264 265 266 267 268 269
	srcu_read_unlock(&netpoll_srcu, idx);
}
EXPORT_SYMBOL(netpoll_rx_disable);

void netpoll_rx_enable(struct net_device *dev)
{
	struct netpoll_info *ni;
	rcu_read_lock();
	ni = rcu_dereference(dev->npinfo);
	if (ni)
270
		up(&ni->dev_lock);
271 272 273 274
	rcu_read_unlock();
}
EXPORT_SYMBOL(netpoll_rx_enable);

L
Linus Torvalds 已提交
275 276 277 278 279
static void refill_skbs(void)
{
	struct sk_buff *skb;
	unsigned long flags;

280 281
	spin_lock_irqsave(&skb_pool.lock, flags);
	while (skb_pool.qlen < MAX_SKBS) {
L
Linus Torvalds 已提交
282 283 284 285
		skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC);
		if (!skb)
			break;

286
		__skb_queue_tail(&skb_pool, skb);
L
Linus Torvalds 已提交
287
	}
288
	spin_unlock_irqrestore(&skb_pool.lock, flags);
L
Linus Torvalds 已提交
289 290
}

291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318
static void zap_completion_queue(void)
{
	unsigned long flags;
	struct softnet_data *sd = &get_cpu_var(softnet_data);

	if (sd->completion_queue) {
		struct sk_buff *clist;

		local_irq_save(flags);
		clist = sd->completion_queue;
		sd->completion_queue = NULL;
		local_irq_restore(flags);

		while (clist != NULL) {
			struct sk_buff *skb = clist;
			clist = clist->next;
			if (skb->destructor) {
				atomic_inc(&skb->users);
				dev_kfree_skb_any(skb); /* put this one back */
			} else {
				__kfree_skb(skb);
			}
		}
	}

	put_cpu_var(softnet_data);
}

319
static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
L
Linus Torvalds 已提交
320
{
321 322
	int count = 0;
	struct sk_buff *skb;
L
Linus Torvalds 已提交
323

324
	zap_completion_queue();
325
	refill_skbs();
L
Linus Torvalds 已提交
326 327 328
repeat:

	skb = alloc_skb(len, GFP_ATOMIC);
329 330
	if (!skb)
		skb = skb_dequeue(&skb_pool);
L
Linus Torvalds 已提交
331 332

	if (!skb) {
333
		if (++count < 10) {
334
			netpoll_poll_dev(np->dev);
335
			goto repeat;
L
Linus Torvalds 已提交
336
		}
337
		return NULL;
L
Linus Torvalds 已提交
338 339 340 341 342 343 344
	}

	atomic_set(&skb->users, 1);
	skb_reserve(skb, reserve);
	return skb;
}

345 346 347 348 349 350 351 352 353 354 355
static int netpoll_owner_active(struct net_device *dev)
{
	struct napi_struct *napi;

	list_for_each_entry(napi, &dev->napi_list, dev_list) {
		if (napi->poll_owner == smp_processor_id())
			return 1;
	}
	return 0;
}

356
/* call with IRQ disabled */
357 358
void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
			     struct net_device *dev)
L
Linus Torvalds 已提交
359
{
S
Stephen Hemminger 已提交
360 361
	int status = NETDEV_TX_BUSY;
	unsigned long tries;
362
	const struct net_device_ops *ops = dev->netdev_ops;
H
Herbert Xu 已提交
363
	/* It is up to the caller to keep npinfo alive. */
364
	struct netpoll_info *npinfo;
S
Stephen Hemminger 已提交
365

366 367 368
	WARN_ON_ONCE(!irqs_disabled());

	npinfo = rcu_dereference_bh(np->dev->npinfo);
369 370 371 372
	if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
		__kfree_skb(skb);
		return;
	}
S
Stephen Hemminger 已提交
373 374

	/* don't get messages out of order, and no recursion */
375
	if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
376
		struct netdev_queue *txq;
377

378
		txq = netdev_pick_tx(dev, skb);
379

380 381 382
		/* try until next clock tick */
		for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
		     tries > 0; --tries) {
383
			if (__netif_tx_trylock(txq)) {
384
				if (!netif_xmit_stopped(txq)) {
385
					if (vlan_tx_tag_present(skb) &&
386 387 388
					    !vlan_hw_offload_capable(netif_skb_features(skb),
								     skb->vlan_proto)) {
						skb = __vlan_put_tag(skb, skb->vlan_proto, vlan_tx_tag_get(skb));
389 390 391 392 393
						if (unlikely(!skb))
							break;
						skb->vlan_tci = 0;
					}

394
					status = ops->ndo_start_xmit(skb, dev);
E
Eric Dumazet 已提交
395 396 397
					if (status == NETDEV_TX_OK)
						txq_trans_update(txq);
				}
398
				__netif_tx_unlock(txq);
399 400 401 402 403

				if (status == NETDEV_TX_OK)
					break;

			}
404 405

			/* tickle device maybe there is some cleanup */
406
			netpoll_poll_dev(np->dev);
407 408

			udelay(USEC_PER_POLL);
M
Matt Mackall 已提交
409
		}
410 411

		WARN_ONCE(!irqs_disabled(),
412
			"netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pF)\n",
413 414
			dev->name, ops->ndo_start_xmit);

L
Linus Torvalds 已提交
415 416
	}

S
Stephen Hemminger 已提交
417
	if (status != NETDEV_TX_OK) {
S
Stephen Hemminger 已提交
418
		skb_queue_tail(&npinfo->txq, skb);
419
		schedule_delayed_work(&npinfo->tx_work,0);
L
Linus Torvalds 已提交
420 421
	}
}
422
EXPORT_SYMBOL(netpoll_send_skb_on_dev);
L
Linus Torvalds 已提交
423 424 425

void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
{
426
	int total_len, ip_len, udp_len;
L
Linus Torvalds 已提交
427 428 429 430
	struct sk_buff *skb;
	struct udphdr *udph;
	struct iphdr *iph;
	struct ethhdr *eth;
431
	static atomic_t ip_ident;
C
Cong Wang 已提交
432
	struct ipv6hdr *ip6h;
L
Linus Torvalds 已提交
433 434

	udp_len = len + sizeof(*udph);
C
Cong Wang 已提交
435 436 437
	if (np->ipv6)
		ip_len = udp_len + sizeof(*ip6h);
	else
C
Cong Wang 已提交
438 439
		ip_len = udp_len + sizeof(*iph);

440
	total_len = ip_len + LL_RESERVED_SPACE(np->dev);
L
Linus Torvalds 已提交
441

442 443
	skb = find_skb(np, total_len + np->dev->needed_tailroom,
		       total_len - len);
L
Linus Torvalds 已提交
444 445 446
	if (!skb)
		return;

447
	skb_copy_to_linear_data(skb, msg, len);
448
	skb_put(skb, len);
L
Linus Torvalds 已提交
449

450 451 452
	skb_push(skb, sizeof(*udph));
	skb_reset_transport_header(skb);
	udph = udp_hdr(skb);
L
Linus Torvalds 已提交
453 454 455
	udph->source = htons(np->local_port);
	udph->dest = htons(np->remote_port);
	udph->len = htons(udp_len);
C
Cong Wang 已提交
456

C
Cong Wang 已提交
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
	if (np->ipv6) {
		udph->check = 0;
		udph->check = csum_ipv6_magic(&np->local_ip.in6,
					      &np->remote_ip.in6,
					      udp_len, IPPROTO_UDP,
					      csum_partial(udph, udp_len, 0));
		if (udph->check == 0)
			udph->check = CSUM_MANGLED_0;

		skb_push(skb, sizeof(*ip6h));
		skb_reset_network_header(skb);
		ip6h = ipv6_hdr(skb);

		/* ip6h->version = 6; ip6h->priority = 0; */
		put_unaligned(0x60, (unsigned char *)ip6h);
		ip6h->flow_lbl[0] = 0;
		ip6h->flow_lbl[1] = 0;
		ip6h->flow_lbl[2] = 0;

		ip6h->payload_len = htons(sizeof(struct udphdr) + len);
		ip6h->nexthdr = IPPROTO_UDP;
		ip6h->hop_limit = 32;
		ip6h->saddr = np->local_ip.in6;
		ip6h->daddr = np->remote_ip.in6;

		eth = (struct ethhdr *) skb_push(skb, ETH_HLEN);
		skb_reset_mac_header(skb);
		skb->protocol = eth->h_proto = htons(ETH_P_IPV6);
	} else {
C
Cong Wang 已提交
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
		udph->check = 0;
		udph->check = csum_tcpudp_magic(np->local_ip.ip,
						np->remote_ip.ip,
						udp_len, IPPROTO_UDP,
						csum_partial(udph, udp_len, 0));
		if (udph->check == 0)
			udph->check = CSUM_MANGLED_0;

		skb_push(skb, sizeof(*iph));
		skb_reset_network_header(skb);
		iph = ip_hdr(skb);

		/* iph->version = 4; iph->ihl = 5; */
		put_unaligned(0x45, (unsigned char *)iph);
		iph->tos      = 0;
		put_unaligned(htons(ip_len), &(iph->tot_len));
		iph->id       = htons(atomic_inc_return(&ip_ident));
		iph->frag_off = 0;
		iph->ttl      = 64;
		iph->protocol = IPPROTO_UDP;
		iph->check    = 0;
		put_unaligned(np->local_ip.ip, &(iph->saddr));
		put_unaligned(np->remote_ip.ip, &(iph->daddr));
		iph->check    = ip_fast_csum((unsigned char *)iph, iph->ihl);

		eth = (struct ethhdr *) skb_push(skb, ETH_HLEN);
		skb_reset_mac_header(skb);
		skb->protocol = eth->h_proto = htons(ETH_P_IP);
	}

516 517
	memcpy(eth->h_source, np->dev->dev_addr, ETH_ALEN);
	memcpy(eth->h_dest, np->remote_mac, ETH_ALEN);
L
Linus Torvalds 已提交
518 519 520 521 522

	skb->dev = np->dev;

	netpoll_send_skb(np, skb);
}
E
Eric Dumazet 已提交
523
EXPORT_SYMBOL(netpoll_send_udp);
L
Linus Torvalds 已提交
524

C
Cong Wang 已提交
525
static void netpoll_neigh_reply(struct sk_buff *skb, struct netpoll_info *npinfo)
L
Linus Torvalds 已提交
526
{
C
Cong Wang 已提交
527
	int size, type = ARPOP_REPLY;
A
Al Viro 已提交
528
	__be32 sip, tip;
529
	unsigned char *sha;
L
Linus Torvalds 已提交
530
	struct sk_buff *send_skb;
531 532
	struct netpoll *np, *tmp;
	unsigned long flags;
533
	int hlen, tlen;
C
Cong Wang 已提交
534
	int hits = 0, proto;
535 536 537 538 539 540 541 542 543 544 545 546

	if (list_empty(&npinfo->rx_np))
		return;

	/* Before checking the packet, we do some early
	   inspection whether this is interesting at all */
	spin_lock_irqsave(&npinfo->rx_lock, flags);
	list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
		if (np->dev == skb->dev)
			hits++;
	}
	spin_unlock_irqrestore(&npinfo->rx_lock, flags);
L
Linus Torvalds 已提交
547

548 549
	/* No netpoll struct is using this dev */
	if (!hits)
550
		return;
L
Linus Torvalds 已提交
551

C
Cong Wang 已提交
552
	proto = ntohs(eth_hdr(skb)->h_proto);
553
	if (proto == ETH_P_ARP) {
C
Cong Wang 已提交
554 555
		struct arphdr *arp;
		unsigned char *arp_ptr;
C
Cong Wang 已提交
556 557 558
		/* No arp on this interface */
		if (skb->dev->flags & IFF_NOARP)
			return;
L
Linus Torvalds 已提交
559

C
Cong Wang 已提交
560 561
		if (!pskb_may_pull(skb, arp_hdr_len(skb->dev)))
			return;
L
Linus Torvalds 已提交
562

C
Cong Wang 已提交
563 564 565
		skb_reset_network_header(skb);
		skb_reset_transport_header(skb);
		arp = arp_hdr(skb);
L
Linus Torvalds 已提交
566

C
Cong Wang 已提交
567 568 569 570 571
		if ((arp->ar_hrd != htons(ARPHRD_ETHER) &&
		     arp->ar_hrd != htons(ARPHRD_IEEE802)) ||
		    arp->ar_pro != htons(ETH_P_IP) ||
		    arp->ar_op != htons(ARPOP_REQUEST))
			return;
L
Linus Torvalds 已提交
572

C
Cong Wang 已提交
573 574 575 576 577 578 579 580 581 582
		arp_ptr = (unsigned char *)(arp+1);
		/* save the location of the src hw addr */
		sha = arp_ptr;
		arp_ptr += skb->dev->addr_len;
		memcpy(&sip, arp_ptr, 4);
		arp_ptr += 4;
		/* If we actually cared about dst hw addr,
		   it would get copied here */
		arp_ptr += skb->dev->addr_len;
		memcpy(&tip, arp_ptr, 4);
L
Linus Torvalds 已提交
583

C
Cong Wang 已提交
584 585 586
		/* Should we ignore arp? */
		if (ipv4_is_loopback(tip) || ipv4_is_multicast(tip))
			return;
L
Linus Torvalds 已提交
587

C
Cong Wang 已提交
588
		size = arp_hdr_len(skb->dev);
L
Linus Torvalds 已提交
589

C
Cong Wang 已提交
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
		spin_lock_irqsave(&npinfo->rx_lock, flags);
		list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
			if (tip != np->local_ip.ip)
				continue;

			hlen = LL_RESERVED_SPACE(np->dev);
			tlen = np->dev->needed_tailroom;
			send_skb = find_skb(np, size + hlen + tlen, hlen);
			if (!send_skb)
				continue;

			skb_reset_network_header(send_skb);
			arp = (struct arphdr *) skb_put(send_skb, size);
			send_skb->dev = skb->dev;
			send_skb->protocol = htons(ETH_P_ARP);

			/* Fill the device header for the ARP frame */
C
Cong Wang 已提交
607
			if (dev_hard_header(send_skb, skb->dev, ETH_P_ARP,
C
Cong Wang 已提交
608 609 610 611 612
					    sha, np->dev->dev_addr,
					    send_skb->len) < 0) {
				kfree_skb(send_skb);
				continue;
			}
L
Linus Torvalds 已提交
613

C
Cong Wang 已提交
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
			/*
			 * Fill out the arp protocol part.
			 *
			 * we only support ethernet device type,
			 * which (according to RFC 1390) should
			 * always equal 1 (Ethernet).
			 */

			arp->ar_hrd = htons(np->dev->type);
			arp->ar_pro = htons(ETH_P_IP);
			arp->ar_hln = np->dev->addr_len;
			arp->ar_pln = 4;
			arp->ar_op = htons(type);

			arp_ptr = (unsigned char *)(arp + 1);
			memcpy(arp_ptr, np->dev->dev_addr, np->dev->addr_len);
			arp_ptr += np->dev->addr_len;
			memcpy(arp_ptr, &tip, 4);
			arp_ptr += 4;
			memcpy(arp_ptr, sha, np->dev->addr_len);
			arp_ptr += np->dev->addr_len;
			memcpy(arp_ptr, &sip, 4);

			netpoll_send_skb(np, send_skb);

			/* If there are several rx_hooks for the same address,
			   we're fine by sending a single reply */
			break;
642
		}
C
Cong Wang 已提交
643
		spin_unlock_irqrestore(&npinfo->rx_lock, flags);
C
Cong Wang 已提交
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
	} else if( proto == ETH_P_IPV6) {
#if IS_ENABLED(CONFIG_IPV6)
		struct nd_msg *msg;
		u8 *lladdr = NULL;
		struct ipv6hdr *hdr;
		struct icmp6hdr *icmp6h;
		const struct in6_addr *saddr;
		const struct in6_addr *daddr;
		struct inet6_dev *in6_dev = NULL;
		struct in6_addr *target;

		in6_dev = in6_dev_get(skb->dev);
		if (!in6_dev || !in6_dev->cnf.accept_ra)
			return;

		if (!pskb_may_pull(skb, skb->len))
			return;

		msg = (struct nd_msg *)skb_transport_header(skb);

		__skb_push(skb, skb->data - skb_transport_header(skb));

		if (ipv6_hdr(skb)->hop_limit != 255)
			return;
		if (msg->icmph.icmp6_code != 0)
			return;
		if (msg->icmph.icmp6_type != NDISC_NEIGHBOUR_SOLICITATION)
			return;

		saddr = &ipv6_hdr(skb)->saddr;
		daddr = &ipv6_hdr(skb)->daddr;

		size = sizeof(struct icmp6hdr) + sizeof(struct in6_addr);

		spin_lock_irqsave(&npinfo->rx_lock, flags);
		list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
680
			if (!ipv6_addr_equal(daddr, &np->local_ip.in6))
C
Cong Wang 已提交
681 682 683 684 685 686 687 688 689 690 691 692
				continue;

			hlen = LL_RESERVED_SPACE(np->dev);
			tlen = np->dev->needed_tailroom;
			send_skb = find_skb(np, size + hlen + tlen, hlen);
			if (!send_skb)
				continue;

			send_skb->protocol = htons(ETH_P_IPV6);
			send_skb->dev = skb->dev;

			skb_reset_network_header(send_skb);
693
			hdr = (struct ipv6hdr *) skb_put(send_skb, sizeof(struct ipv6hdr));
C
Cong Wang 已提交
694 695 696 697 698 699 700
			*(__be32*)hdr = htonl(0x60000000);
			hdr->payload_len = htons(size);
			hdr->nexthdr = IPPROTO_ICMPV6;
			hdr->hop_limit = 255;
			hdr->saddr = *saddr;
			hdr->daddr = *daddr;

701
			icmp6h = (struct icmp6hdr *) skb_put(send_skb, sizeof(struct icmp6hdr));
C
Cong Wang 已提交
702 703 704
			icmp6h->icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT;
			icmp6h->icmp6_router = 0;
			icmp6h->icmp6_solicited = 1;
705 706

			target = (struct in6_addr *) skb_put(send_skb, sizeof(struct in6_addr));
C
Cong Wang 已提交
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
			*target = msg->target;
			icmp6h->icmp6_cksum = csum_ipv6_magic(saddr, daddr, size,
							      IPPROTO_ICMPV6,
							      csum_partial(icmp6h,
									   size, 0));

			if (dev_hard_header(send_skb, skb->dev, ETH_P_IPV6,
					    lladdr, np->dev->dev_addr,
					    send_skb->len) < 0) {
				kfree_skb(send_skb);
				continue;
			}

			netpoll_send_skb(np, send_skb);

			/* If there are several rx_hooks for the same address,
			   we're fine by sending a single reply */
			break;
		}
		spin_unlock_irqrestore(&npinfo->rx_lock, flags);
#endif
728
	}
L
Linus Torvalds 已提交
729 730
}

C
Cong Wang 已提交
731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756
static bool pkt_is_ns(struct sk_buff *skb)
{
	struct nd_msg *msg;
	struct ipv6hdr *hdr;

	if (skb->protocol != htons(ETH_P_ARP))
		return false;
	if (!pskb_may_pull(skb, sizeof(struct ipv6hdr) + sizeof(struct nd_msg)))
		return false;

	msg = (struct nd_msg *)skb_transport_header(skb);
	__skb_push(skb, skb->data - skb_transport_header(skb));
	hdr = ipv6_hdr(skb);

	if (hdr->nexthdr != IPPROTO_ICMPV6)
		return false;
	if (hdr->hop_limit != 255)
		return false;
	if (msg->icmph.icmp6_code != 0)
		return false;
	if (msg->icmph.icmp6_type != NDISC_NEIGHBOUR_SOLICITATION)
		return false;

	return true;
}

757
int __netpoll_rx(struct sk_buff *skb, struct netpoll_info *npinfo)
L
Linus Torvalds 已提交
758 759
{
	int proto, len, ulen;
760
	int hits = 0;
761
	const struct iphdr *iph;
L
Linus Torvalds 已提交
762
	struct udphdr *uh;
763
	struct netpoll *np, *tmp;
764

765
	if (list_empty(&npinfo->rx_np))
L
Linus Torvalds 已提交
766
		goto out;
767

L
Linus Torvalds 已提交
768 769 770
	if (skb->dev->type != ARPHRD_ETHER)
		goto out;

771
	/* check if netpoll clients need ARP */
C
Cong Wang 已提交
772 773 774 775
	if (skb->protocol == htons(ETH_P_ARP) && atomic_read(&trapped)) {
		skb_queue_tail(&npinfo->neigh_tx, skb);
		return 1;
	} else if (pkt_is_ns(skb) && atomic_read(&trapped)) {
C
Cong Wang 已提交
776
		skb_queue_tail(&npinfo->neigh_tx, skb);
L
Linus Torvalds 已提交
777 778 779
		return 1;
	}

780 781 782 783 784 785
	if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
		skb = vlan_untag(skb);
		if (unlikely(!skb))
			goto out;
	}

L
Linus Torvalds 已提交
786
	proto = ntohs(eth_hdr(skb)->h_proto);
C
Cong Wang 已提交
787
	if (proto != ETH_P_IP && proto != ETH_P_IPV6)
L
Linus Torvalds 已提交
788 789 790 791 792 793
		goto out;
	if (skb->pkt_type == PACKET_OTHERHOST)
		goto out;
	if (skb_shared(skb))
		goto out;

C
Cong Wang 已提交
794 795 796 797 798 799 800 801 802 803 804
	if (proto == ETH_P_IP) {
		if (!pskb_may_pull(skb, sizeof(struct iphdr)))
			goto out;
		iph = (struct iphdr *)skb->data;
		if (iph->ihl < 5 || iph->version != 4)
			goto out;
		if (!pskb_may_pull(skb, iph->ihl*4))
			goto out;
		iph = (struct iphdr *)skb->data;
		if (ip_fast_csum((u8 *)iph, iph->ihl) != 0)
			goto out;
805

C
Cong Wang 已提交
806 807 808
		len = ntohs(iph->tot_len);
		if (skb->len < len || len < iph->ihl*4)
			goto out;
L
Linus Torvalds 已提交
809

C
Cong Wang 已提交
810 811 812 813 814 815
		/*
		 * Our transport medium may have padded the buffer out.
		 * Now We trim to the true length of the frame.
		 */
		if (pskb_trim_rcsum(skb, len))
			goto out;
L
Linus Torvalds 已提交
816

C
Cong Wang 已提交
817 818 819
		iph = (struct iphdr *)skb->data;
		if (iph->protocol != IPPROTO_UDP)
			goto out;
L
Linus Torvalds 已提交
820

C
Cong Wang 已提交
821 822 823
		len -= iph->ihl*4;
		uh = (struct udphdr *)(((char *)iph) + iph->ihl*4);
		ulen = ntohs(uh->len);
824

C
Cong Wang 已提交
825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
		if (ulen != len)
			goto out;
		if (checksum_udp(skb, uh, ulen, iph->saddr, iph->daddr))
			goto out;
		list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
			if (np->local_ip.ip && np->local_ip.ip != iph->daddr)
				continue;
			if (np->remote_ip.ip && np->remote_ip.ip != iph->saddr)
				continue;
			if (np->local_port && np->local_port != ntohs(uh->dest))
				continue;

			np->rx_hook(np, ntohs(uh->source),
				       (char *)(uh+1),
				       ulen - sizeof(struct udphdr));
			hits++;
		}
C
Cong Wang 已提交
842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
	} else {
#if IS_ENABLED(CONFIG_IPV6)
		const struct ipv6hdr *ip6h;

		if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
			goto out;
		ip6h = (struct ipv6hdr *)skb->data;
		if (ip6h->version != 6)
			goto out;
		len = ntohs(ip6h->payload_len);
		if (!len)
			goto out;
		if (len + sizeof(struct ipv6hdr) > skb->len)
			goto out;
		if (pskb_trim_rcsum(skb, len + sizeof(struct ipv6hdr)))
			goto out;
		ip6h = ipv6_hdr(skb);
		if (!pskb_may_pull(skb, sizeof(struct udphdr)))
			goto out;
		uh = udp_hdr(skb);
		ulen = ntohs(uh->len);
		if (ulen != skb->len)
			goto out;
		if (udp6_csum_init(skb, uh, IPPROTO_UDP))
			goto out;
		list_for_each_entry_safe(np, tmp, &npinfo->rx_np, rx) {
868
			if (!ipv6_addr_equal(&np->local_ip.in6, &ip6h->daddr))
C
Cong Wang 已提交
869
				continue;
870
			if (!ipv6_addr_equal(&np->remote_ip.in6, &ip6h->saddr))
C
Cong Wang 已提交
871 872 873 874 875 876 877 878 879 880
				continue;
			if (np->local_port && np->local_port != ntohs(uh->dest))
				continue;

			np->rx_hook(np, ntohs(uh->source),
				       (char *)(uh+1),
				       ulen - sizeof(struct udphdr));
			hits++;
		}
#endif
881 882 883 884
	}

	if (!hits)
		goto out;
L
Linus Torvalds 已提交
885 886 887 888 889 890 891 892 893 894 895 896 897

	kfree_skb(skb);
	return 1;

out:
	if (atomic_read(&trapped)) {
		kfree_skb(skb);
		return 1;
	}

	return 0;
}

898 899
void netpoll_print_options(struct netpoll *np)
{
900
	np_info(np, "local port %d\n", np->local_port);
C
Cong Wang 已提交
901 902 903
	if (np->ipv6)
		np_info(np, "local IPv6 address %pI6c\n", &np->local_ip.in6);
	else
C
Cong Wang 已提交
904
		np_info(np, "local IPv4 address %pI4\n", &np->local_ip.ip);
905 906
	np_info(np, "interface '%s'\n", np->dev_name);
	np_info(np, "remote port %d\n", np->remote_port);
C
Cong Wang 已提交
907 908 909
	if (np->ipv6)
		np_info(np, "remote IPv6 address %pI6c\n", &np->remote_ip.in6);
	else
C
Cong Wang 已提交
910
		np_info(np, "remote IPv4 address %pI4\n", &np->remote_ip.ip);
911
	np_info(np, "remote ethernet address %pM\n", np->remote_mac);
912
}
E
Eric Dumazet 已提交
913
EXPORT_SYMBOL(netpoll_print_options);
914

C
Cong Wang 已提交
915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
{
	const char *end;

	if (!strchr(str, ':') &&
	    in4_pton(str, -1, (void *)addr, -1, &end) > 0) {
		if (!*end)
			return 0;
	}
	if (in6_pton(str, -1, addr->in6.s6_addr, -1, &end) > 0) {
#if IS_ENABLED(CONFIG_IPV6)
		if (!*end)
			return 1;
#else
		return -1;
#endif
	}
	return -1;
}

L
Linus Torvalds 已提交
935 936 937
int netpoll_parse_options(struct netpoll *np, char *opt)
{
	char *cur=opt, *delim;
C
Cong Wang 已提交
938
	int ipv6;
L
Linus Torvalds 已提交
939

940
	if (*cur != '@') {
L
Linus Torvalds 已提交
941 942
		if ((delim = strchr(cur, '@')) == NULL)
			goto parse_failed;
943
		*delim = 0;
944 945
		if (kstrtou16(cur, 10, &np->local_port))
			goto parse_failed;
946
		cur = delim;
L
Linus Torvalds 已提交
947 948 949
	}
	cur++;

950
	if (*cur != '/') {
L
Linus Torvalds 已提交
951 952
		if ((delim = strchr(cur, '/')) == NULL)
			goto parse_failed;
953
		*delim = 0;
C
Cong Wang 已提交
954 955 956 957 958
		ipv6 = netpoll_parse_ip_addr(cur, &np->local_ip);
		if (ipv6 < 0)
			goto parse_failed;
		else
			np->ipv6 = (bool)ipv6;
959
		cur = delim;
L
Linus Torvalds 已提交
960 961 962
	}
	cur++;

963
	if (*cur != ',') {
L
Linus Torvalds 已提交
964 965 966
		/* parse out dev name */
		if ((delim = strchr(cur, ',')) == NULL)
			goto parse_failed;
967
		*delim = 0;
L
Linus Torvalds 已提交
968
		strlcpy(np->dev_name, cur, sizeof(np->dev_name));
969
		cur = delim;
L
Linus Torvalds 已提交
970 971 972
	}
	cur++;

973
	if (*cur != '@') {
L
Linus Torvalds 已提交
974 975 976
		/* dst port */
		if ((delim = strchr(cur, '@')) == NULL)
			goto parse_failed;
977
		*delim = 0;
978
		if (*cur == ' ' || *cur == '\t')
979
			np_info(np, "warning: whitespace is not allowed\n");
980 981
		if (kstrtou16(cur, 10, &np->remote_port))
			goto parse_failed;
982
		cur = delim;
L
Linus Torvalds 已提交
983 984 985 986 987 988
	}
	cur++;

	/* dst ip */
	if ((delim = strchr(cur, '/')) == NULL)
		goto parse_failed;
989
	*delim = 0;
C
Cong Wang 已提交
990 991 992 993 994 995 996
	ipv6 = netpoll_parse_ip_addr(cur, &np->remote_ip);
	if (ipv6 < 0)
		goto parse_failed;
	else if (np->ipv6 != (bool)ipv6)
		goto parse_failed;
	else
		np->ipv6 = (bool)ipv6;
997
	cur = delim + 1;
L
Linus Torvalds 已提交
998

999
	if (*cur != 0) {
L
Linus Torvalds 已提交
1000
		/* MAC address */
1001
		if (!mac_pton(cur, np->remote_mac))
L
Linus Torvalds 已提交
1002 1003 1004
			goto parse_failed;
	}

1005
	netpoll_print_options(np);
L
Linus Torvalds 已提交
1006 1007 1008 1009

	return 0;

 parse_failed:
1010
	np_info(np, "couldn't parse config at '%s'!\n", cur);
L
Linus Torvalds 已提交
1011 1012
	return -1;
}
E
Eric Dumazet 已提交
1013
EXPORT_SYMBOL(netpoll_parse_options);
L
Linus Torvalds 已提交
1014

1015
int __netpoll_setup(struct netpoll *np, struct net_device *ndev, gfp_t gfp)
L
Linus Torvalds 已提交
1016
{
1017
	struct netpoll_info *npinfo;
H
Herbert Xu 已提交
1018
	const struct net_device_ops *ops;
1019
	unsigned long flags;
S
Stephen Hemminger 已提交
1020
	int err;
L
Linus Torvalds 已提交
1021

1022 1023
	np->dev = ndev;
	strlcpy(np->dev_name, ndev->name, IFNAMSIZ);
1024
	INIT_WORK(&np->cleanup_work, netpoll_async_cleanup);
1025

1026 1027
	if ((ndev->priv_flags & IFF_DISABLE_NETPOLL) ||
	    !ndev->netdev_ops->ndo_poll_controller) {
1028 1029
		np_err(np, "%s doesn't support polling, aborting\n",
		       np->dev_name);
1030 1031 1032 1033 1034
		err = -ENOTSUPP;
		goto out;
	}

	if (!ndev->npinfo) {
1035
		npinfo = kmalloc(sizeof(*npinfo), gfp);
1036 1037 1038 1039 1040 1041 1042 1043 1044
		if (!npinfo) {
			err = -ENOMEM;
			goto out;
		}

		npinfo->rx_flags = 0;
		INIT_LIST_HEAD(&npinfo->rx_np);

		spin_lock_init(&npinfo->rx_lock);
1045
		sema_init(&npinfo->dev_lock, 1);
C
Cong Wang 已提交
1046
		skb_queue_head_init(&npinfo->neigh_tx);
1047 1048 1049 1050 1051 1052 1053
		skb_queue_head_init(&npinfo->txq);
		INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);

		atomic_set(&npinfo->refcnt, 1);

		ops = np->dev->netdev_ops;
		if (ops->ndo_netpoll_setup) {
1054
			err = ops->ndo_netpoll_setup(ndev, npinfo, gfp);
1055 1056 1057 1058
			if (err)
				goto free_npinfo;
		}
	} else {
N
Neil Horman 已提交
1059
		npinfo = rtnl_dereference(ndev->npinfo);
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
		atomic_inc(&npinfo->refcnt);
	}

	npinfo->netpoll = np;

	if (np->rx_hook) {
		spin_lock_irqsave(&npinfo->rx_lock, flags);
		npinfo->rx_flags |= NETPOLL_RX_ENABLED;
		list_add_tail(&np->rx, &npinfo->rx_np);
		spin_unlock_irqrestore(&npinfo->rx_lock, flags);
	}

	/* last thing to do is link it to the net device structure */
1073
	rcu_assign_pointer(ndev->npinfo, npinfo);
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089

	return 0;

free_npinfo:
	kfree(npinfo);
out:
	return err;
}
EXPORT_SYMBOL_GPL(__netpoll_setup);

int netpoll_setup(struct netpoll *np)
{
	struct net_device *ndev = NULL;
	struct in_device *in_dev;
	int err;

1090
	rtnl_lock();
1091 1092 1093 1094
	if (np->dev_name) {
		struct net *net = current->nsproxy->net_ns;
		ndev = __dev_get_by_name(net, np->dev_name);
	}
L
Linus Torvalds 已提交
1095
	if (!ndev) {
1096
		np_err(np, "%s doesn't exist, aborting\n", np->dev_name);
1097 1098
		err = -ENODEV;
		goto unlock;
L
Linus Torvalds 已提交
1099
	}
1100
	dev_hold(ndev);
L
Linus Torvalds 已提交
1101

1102
	if (netdev_master_upper_dev_get(ndev)) {
1103
		np_err(np, "%s is a slave device, aborting\n", np->dev_name);
1104 1105
		err = -EBUSY;
		goto put;
1106 1107
	}

L
Linus Torvalds 已提交
1108 1109 1110
	if (!netif_running(ndev)) {
		unsigned long atmost, atleast;

1111
		np_info(np, "device %s not up yet, forcing it\n", np->dev_name);
L
Linus Torvalds 已提交
1112

S
Stephen Hemminger 已提交
1113 1114 1115
		err = dev_open(ndev);

		if (err) {
1116
			np_err(np, "failed to open %s\n", ndev->name);
1117
			goto put;
L
Linus Torvalds 已提交
1118 1119
		}

1120
		rtnl_unlock();
L
Linus Torvalds 已提交
1121
		atleast = jiffies + HZ/10;
1122
		atmost = jiffies + carrier_timeout * HZ;
L
Linus Torvalds 已提交
1123 1124
		while (!netif_carrier_ok(ndev)) {
			if (time_after(jiffies, atmost)) {
1125
				np_notice(np, "timeout waiting for carrier\n");
L
Linus Torvalds 已提交
1126 1127
				break;
			}
1128
			msleep(1);
L
Linus Torvalds 已提交
1129 1130 1131 1132 1133 1134 1135 1136
		}

		/* If carrier appears to come up instantly, we don't
		 * trust it and pause so that we don't pump all our
		 * queued console messages into the bitbucket.
		 */

		if (time_before(jiffies, atleast)) {
1137
			np_notice(np, "carrier detect appears untrustworthy, waiting 4 seconds\n");
L
Linus Torvalds 已提交
1138 1139
			msleep(4000);
		}
1140
		rtnl_lock();
L
Linus Torvalds 已提交
1141 1142
	}

C
Cong Wang 已提交
1143 1144
	if (!np->local_ip.ip) {
		if (!np->ipv6) {
1145
			in_dev = __in_dev_get_rtnl(ndev);
C
Cong Wang 已提交
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155

			if (!in_dev || !in_dev->ifa_list) {
				np_err(np, "no IP address for %s, aborting\n",
				       np->dev_name);
				err = -EDESTADDRREQ;
				goto put;
			}

			np->local_ip.ip = in_dev->ifa_list->ifa_local;
			np_info(np, "local IP %pI4\n", &np->local_ip.ip);
C
Cong Wang 已提交
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
		} else {
#if IS_ENABLED(CONFIG_IPV6)
			struct inet6_dev *idev;

			err = -EDESTADDRREQ;
			idev = __in6_dev_get(ndev);
			if (idev) {
				struct inet6_ifaddr *ifp;

				read_lock_bh(&idev->lock);
				list_for_each_entry(ifp, &idev->addr_list, if_list) {
					if (ipv6_addr_type(&ifp->addr) & IPV6_ADDR_LINKLOCAL)
						continue;
					np->local_ip.in6 = ifp->addr;
					err = 0;
					break;
				}
				read_unlock_bh(&idev->lock);
			}
			if (err) {
				np_err(np, "no IPv6 address for %s, aborting\n",
				       np->dev_name);
				goto put;
			} else
				np_info(np, "local IPv6 %pI6c\n", &np->local_ip.in6);
#else
			np_err(np, "IPv6 is not supported %s, aborting\n",
			       np->dev_name);
1184
			err = -EINVAL;
C
Cong Wang 已提交
1185 1186
			goto put;
#endif
L
Linus Torvalds 已提交
1187 1188 1189
		}
	}

1190 1191 1192
	/* fill up the skb queue */
	refill_skbs();

1193
	err = __netpoll_setup(np, ndev, GFP_KERNEL);
1194 1195 1196
	if (err)
		goto put;

1197
	rtnl_unlock();
L
Linus Torvalds 已提交
1198 1199
	return 0;

1200
put:
L
Linus Torvalds 已提交
1201
	dev_put(ndev);
1202 1203
unlock:
	rtnl_unlock();
S
Stephen Hemminger 已提交
1204
	return err;
L
Linus Torvalds 已提交
1205
}
E
Eric Dumazet 已提交
1206
EXPORT_SYMBOL(netpoll_setup);
L
Linus Torvalds 已提交
1207

1208 1209
static int __init netpoll_init(void)
{
1210 1211 1212 1213 1214
	skb_queue_head_init(&skb_pool);
	return 0;
}
core_initcall(netpoll_init);

1215 1216 1217 1218 1219
static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
{
	struct netpoll_info *npinfo =
			container_of(rcu_head, struct netpoll_info, rcu);

C
Cong Wang 已提交
1220
	skb_queue_purge(&npinfo->neigh_tx);
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232
	skb_queue_purge(&npinfo->txq);

	/* we can't call cancel_delayed_work_sync here, as we are in softirq */
	cancel_delayed_work(&npinfo->tx_work);

	/* clean after last, unfinished work */
	__skb_queue_purge(&npinfo->txq);
	/* now cancel it again */
	cancel_delayed_work(&npinfo->tx_work);
	kfree(npinfo);
}

1233
void __netpoll_cleanup(struct netpoll *np)
L
Linus Torvalds 已提交
1234
{
1235 1236 1237
	struct netpoll_info *npinfo;
	unsigned long flags;

N
Neil Horman 已提交
1238 1239 1240 1241 1242
	/* rtnl_dereference would be preferable here but
	 * rcu_cleanup_netpoll path can put us in here safely without
	 * holding the rtnl, so plain rcu_dereference it is
	 */
	npinfo = rtnl_dereference(np->dev->npinfo);
1243
	if (!npinfo)
1244
		return;
S
Stephen Hemminger 已提交
1245

1246 1247 1248 1249 1250 1251 1252
	if (!list_empty(&npinfo->rx_np)) {
		spin_lock_irqsave(&npinfo->rx_lock, flags);
		list_del(&np->rx);
		if (list_empty(&npinfo->rx_np))
			npinfo->rx_flags &= ~NETPOLL_RX_ENABLED;
		spin_unlock_irqrestore(&npinfo->rx_lock, flags);
	}
H
Herbert Xu 已提交
1253

1254 1255
	synchronize_srcu(&netpoll_srcu);

1256 1257
	if (atomic_dec_and_test(&npinfo->refcnt)) {
		const struct net_device_ops *ops;
H
Herbert Xu 已提交
1258

1259 1260 1261
		ops = np->dev->netdev_ops;
		if (ops->ndo_netpoll_cleanup)
			ops->ndo_netpoll_cleanup(np->dev);
H
Herbert Xu 已提交
1262

1263
		rcu_assign_pointer(np->dev->npinfo, NULL);
1264 1265 1266 1267
		call_rcu_bh(&npinfo->rcu, rcu_cleanup_netpoll_info);
	}
}
EXPORT_SYMBOL_GPL(__netpoll_cleanup);
H
Herbert Xu 已提交
1268

1269
static void netpoll_async_cleanup(struct work_struct *work)
1270
{
1271
	struct netpoll *np = container_of(work, struct netpoll, cleanup_work);
S
Stephen Hemminger 已提交
1272

1273
	rtnl_lock();
1274
	__netpoll_cleanup(np);
1275
	rtnl_unlock();
1276 1277
	kfree(np);
}
S
Stephen Hemminger 已提交
1278

1279
void __netpoll_free_async(struct netpoll *np)
1280
{
1281
	schedule_work(&np->cleanup_work);
1282
}
1283
EXPORT_SYMBOL_GPL(__netpoll_free_async);
1284

1285 1286 1287
void netpoll_cleanup(struct netpoll *np)
{
	rtnl_lock();
1288 1289
	if (!np->dev)
		goto out;
1290 1291
	__netpoll_cleanup(np);
	dev_put(np->dev);
L
Linus Torvalds 已提交
1292
	np->dev = NULL;
1293 1294
out:
	rtnl_unlock();
L
Linus Torvalds 已提交
1295
}
E
Eric Dumazet 已提交
1296
EXPORT_SYMBOL(netpoll_cleanup);
L
Linus Torvalds 已提交
1297 1298 1299 1300 1301

int netpoll_trap(void)
{
	return atomic_read(&trapped);
}
E
Eric Dumazet 已提交
1302
EXPORT_SYMBOL(netpoll_trap);
L
Linus Torvalds 已提交
1303 1304 1305 1306 1307 1308 1309 1310 1311

void netpoll_set_trap(int trap)
{
	if (trap)
		atomic_inc(&trapped);
	else
		atomic_dec(&trapped);
}
EXPORT_SYMBOL(netpoll_set_trap);