netpoll.c 19.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 * Common framework for low-level network console, dump, and debugger code
 *
 * Sep 8 2003  Matt Mackall <mpm@selenic.com>
 *
 * based on the netconsole code from:
 *
 * Copyright (C) 2001  Ingo Molnar <mingo@redhat.com>
 * Copyright (C) 2002  Red Hat, Inc.
 */

12 13
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14
#include <linux/moduleparam.h>
15
#include <linux/kernel.h>
L
Linus Torvalds 已提交
16 17 18
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/string.h>
19
#include <linux/if_arp.h>
L
Linus Torvalds 已提交
20 21 22 23 24 25 26 27
#include <linux/inetdevice.h>
#include <linux/inet.h>
#include <linux/interrupt.h>
#include <linux/netpoll.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
28
#include <linux/slab.h>
29
#include <linux/export.h>
30
#include <linux/if_vlan.h>
L
Linus Torvalds 已提交
31 32
#include <net/tcp.h>
#include <net/udp.h>
C
Cong Wang 已提交
33 34 35
#include <net/addrconf.h>
#include <net/ndisc.h>
#include <net/ip6_checksum.h>
L
Linus Torvalds 已提交
36
#include <asm/unaligned.h>
37
#include <trace/events/napi.h>
L
Linus Torvalds 已提交
38 39 40 41 42 43 44 45 46

/*
 * We maintain a small pool of fully-sized skbs, to make sure the
 * message gets out even in extreme OOM situations.
 */

#define MAX_UDP_CHUNK 1460
#define MAX_SKBS 32

47
static struct sk_buff_head skb_pool;
L
Linus Torvalds 已提交
48

49
DEFINE_STATIC_SRCU(netpoll_srcu);
50

S
Stephen Hemminger 已提交
51
#define USEC_PER_POLL	50
L
Linus Torvalds 已提交
52

J
Joe Perches 已提交
53 54 55 56 57
#define MAX_SKB_SIZE							\
	(sizeof(struct ethhdr) +					\
	 sizeof(struct iphdr) +						\
	 sizeof(struct udphdr) +					\
	 MAX_UDP_CHUNK)
L
Linus Torvalds 已提交
58

59
static void zap_completion_queue(void);
60
static void netpoll_async_cleanup(struct work_struct *work);
L
Linus Torvalds 已提交
61

62 63 64
static unsigned int carrier_timeout = 4;
module_param(carrier_timeout, uint, 0644);

65 66 67 68 69 70 71
#define np_info(np, fmt, ...)				\
	pr_info("%s: " fmt, np->name, ##__VA_ARGS__)
#define np_err(np, fmt, ...)				\
	pr_err("%s: " fmt, np->name, ##__VA_ARGS__)
#define np_notice(np, fmt, ...)				\
	pr_notice("%s: " fmt, np->name, ##__VA_ARGS__)

72 73 74 75 76 77 78 79
static int netpoll_start_xmit(struct sk_buff *skb, struct net_device *dev,
			      struct netdev_queue *txq)
{
	int status = NETDEV_TX_OK;
	netdev_features_t features;

	features = netif_skb_features(skb);

80
	if (skb_vlan_tag_present(skb) &&
81
	    !vlan_hw_offload_capable(features, skb->vlan_proto)) {
82
		skb = __vlan_hwaccel_push_inside(skb);
83 84 85 86 87 88 89 90 91
		if (unlikely(!skb)) {
			/* This is actually a packet drop, but we
			 * don't want the code that calls this
			 * function to try and operate on a NULL skb.
			 */
			goto out;
		}
	}

92
	status = netdev_start_xmit(skb, dev, txq, false);
93 94 95 96 97

out:
	return status;
}

D
David Howells 已提交
98
static void queue_process(struct work_struct *work)
L
Linus Torvalds 已提交
99
{
100 101
	struct netpoll_info *npinfo =
		container_of(work, struct netpoll_info, tx_work.work);
L
Linus Torvalds 已提交
102
	struct sk_buff *skb;
I
Ingo Molnar 已提交
103
	unsigned long flags;
L
Linus Torvalds 已提交
104

S
Stephen Hemminger 已提交
105 106
	while ((skb = skb_dequeue(&npinfo->txq))) {
		struct net_device *dev = skb->dev;
107
		struct netdev_queue *txq;
108
		unsigned int q_index;
L
Linus Torvalds 已提交
109

S
Stephen Hemminger 已提交
110
		if (!netif_device_present(dev) || !netif_running(dev)) {
111
			kfree_skb(skb);
S
Stephen Hemminger 已提交
112 113
			continue;
		}
L
Linus Torvalds 已提交
114

I
Ingo Molnar 已提交
115
		local_irq_save(flags);
116 117 118 119 120 121 122
		/* check if skb->queue_mapping is still valid */
		q_index = skb_get_queue_mapping(skb);
		if (unlikely(q_index >= dev->real_num_tx_queues)) {
			q_index = q_index % dev->real_num_tx_queues;
			skb_set_queue_mapping(skb, q_index);
		}
		txq = netdev_get_tx_queue(dev, q_index);
123
		HARD_TX_LOCK(dev, txq, smp_processor_id());
124
		if (netif_xmit_frozen_or_stopped(txq) ||
125
		    netpoll_start_xmit(skb, dev, txq) != NETDEV_TX_OK) {
S
Stephen Hemminger 已提交
126
			skb_queue_head(&npinfo->txq, skb);
127
			HARD_TX_UNLOCK(dev, txq);
I
Ingo Molnar 已提交
128
			local_irq_restore(flags);
L
Linus Torvalds 已提交
129

130
			schedule_delayed_work(&npinfo->tx_work, HZ/10);
S
Stephen Hemminger 已提交
131 132
			return;
		}
133
		HARD_TX_UNLOCK(dev, txq);
I
Ingo Molnar 已提交
134
		local_irq_restore(flags);
L
Linus Torvalds 已提交
135 136 137 138 139 140 141 142 143 144 145 146 147 148
	}
}

/*
 * Check whether delayed processing was scheduled for our NIC. If so,
 * we attempt to grab the poll lock and use ->poll() to pump the card.
 * If this fails, either we've recursed in ->poll() or it's already
 * running on another CPU.
 *
 * Note: we don't mask interrupts with this lock because we're using
 * trylock here and interrupts are already disabled in the softirq
 * case. Further, we test the poll_owner to avoid recursion on UP
 * systems where the lock doesn't exist.
 */
149
static void poll_one_napi(struct napi_struct *napi)
150
{
151
	int work = 0;
152 153 154 155 156 157

	/* net_rx_action's ->poll() invocations and our's are
	 * synchronized by this test which is only made while
	 * holding the napi->poll_lock.
	 */
	if (!test_bit(NAPI_STATE_SCHED, &napi->state))
158
		return;
159

160 161 162 163 164
	/* If we set this bit but see that it has already been set,
	 * that indicates that napi has been disabled and we need
	 * to abort this operation
	 */
	if (test_and_set_bit(NAPI_STATE_NPSVC, &napi->state))
165
		return;
166

167 168 169 170 171
	/* We explicilty pass the polling call a budget of 0 to
	 * indicate that we are clearing the Tx path only.
	 */
	work = napi->poll(napi, 0);
	WARN_ONCE(work, "%pF exceeded budget in poll\n", napi->poll);
172
	trace_napi_poll(napi, work, 0);
173

174
	clear_bit(NAPI_STATE_NPSVC, &napi->state);
175 176
}

177
static void poll_napi(struct net_device *dev)
L
Linus Torvalds 已提交
178
{
179
	struct napi_struct *napi;
E
Eric Dumazet 已提交
180
	int cpu = smp_processor_id();
L
Linus Torvalds 已提交
181

182
	list_for_each_entry(napi, &dev->napi_list, dev_list) {
E
Eric Dumazet 已提交
183
		if (cmpxchg(&napi->poll_owner, -1, cpu) == -1) {
184
			poll_one_napi(napi);
E
Eric Dumazet 已提交
185
			smp_store_release(&napi->poll_owner, -1);
186
		}
L
Linus Torvalds 已提交
187 188 189
	}
}

190
static void netpoll_poll_dev(struct net_device *dev)
L
Linus Torvalds 已提交
191
{
192
	const struct net_device_ops *ops;
193
	struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
194

195 196 197 198
	/* Don't do any rx activity if the dev_lock mutex is held
	 * the dev_open/close paths use this to block netpoll activity
	 * while changing device state
	 */
199
	if (down_trylock(&ni->dev_lock))
200 201
		return;

202
	if (!netif_running(dev)) {
203
		up(&ni->dev_lock);
204
		return;
205
	}
206 207

	ops = dev->netdev_ops;
208
	if (!ops->ndo_poll_controller) {
209
		up(&ni->dev_lock);
L
Linus Torvalds 已提交
210
		return;
211
	}
L
Linus Torvalds 已提交
212 213

	/* Process pending work on NIC */
214
	ops->ndo_poll_controller(dev);
215

216
	poll_napi(dev);
L
Linus Torvalds 已提交
217

218
	up(&ni->dev_lock);
219

220
	zap_completion_queue();
L
Linus Torvalds 已提交
221 222
}

223
void netpoll_poll_disable(struct net_device *dev)
224 225 226 227 228 229 230
{
	struct netpoll_info *ni;
	int idx;
	might_sleep();
	idx = srcu_read_lock(&netpoll_srcu);
	ni = srcu_dereference(dev->npinfo, &netpoll_srcu);
	if (ni)
231
		down(&ni->dev_lock);
232 233
	srcu_read_unlock(&netpoll_srcu, idx);
}
234
EXPORT_SYMBOL(netpoll_poll_disable);
235

236
void netpoll_poll_enable(struct net_device *dev)
237 238 239 240 241
{
	struct netpoll_info *ni;
	rcu_read_lock();
	ni = rcu_dereference(dev->npinfo);
	if (ni)
242
		up(&ni->dev_lock);
243 244
	rcu_read_unlock();
}
245
EXPORT_SYMBOL(netpoll_poll_enable);
246

L
Linus Torvalds 已提交
247 248 249 250 251
static void refill_skbs(void)
{
	struct sk_buff *skb;
	unsigned long flags;

252 253
	spin_lock_irqsave(&skb_pool.lock, flags);
	while (skb_pool.qlen < MAX_SKBS) {
L
Linus Torvalds 已提交
254 255 256 257
		skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC);
		if (!skb)
			break;

258
		__skb_queue_tail(&skb_pool, skb);
L
Linus Torvalds 已提交
259
	}
260
	spin_unlock_irqrestore(&skb_pool.lock, flags);
L
Linus Torvalds 已提交
261 262
}

263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
static void zap_completion_queue(void)
{
	unsigned long flags;
	struct softnet_data *sd = &get_cpu_var(softnet_data);

	if (sd->completion_queue) {
		struct sk_buff *clist;

		local_irq_save(flags);
		clist = sd->completion_queue;
		sd->completion_queue = NULL;
		local_irq_restore(flags);

		while (clist != NULL) {
			struct sk_buff *skb = clist;
			clist = clist->next;
279
			if (!skb_irq_freeable(skb)) {
280
				refcount_set(&skb->users, 1);
281 282 283 284 285 286 287 288 289 290
				dev_kfree_skb_any(skb); /* put this one back */
			} else {
				__kfree_skb(skb);
			}
		}
	}

	put_cpu_var(softnet_data);
}

291
static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
L
Linus Torvalds 已提交
292
{
293 294
	int count = 0;
	struct sk_buff *skb;
L
Linus Torvalds 已提交
295

296
	zap_completion_queue();
297
	refill_skbs();
L
Linus Torvalds 已提交
298 299 300
repeat:

	skb = alloc_skb(len, GFP_ATOMIC);
301 302
	if (!skb)
		skb = skb_dequeue(&skb_pool);
L
Linus Torvalds 已提交
303 304

	if (!skb) {
305
		if (++count < 10) {
306
			netpoll_poll_dev(np->dev);
307
			goto repeat;
L
Linus Torvalds 已提交
308
		}
309
		return NULL;
L
Linus Torvalds 已提交
310 311
	}

312
	refcount_set(&skb->users, 1);
L
Linus Torvalds 已提交
313 314 315 316
	skb_reserve(skb, reserve);
	return skb;
}

317 318 319 320 321 322 323 324 325 326 327
static int netpoll_owner_active(struct net_device *dev)
{
	struct napi_struct *napi;

	list_for_each_entry(napi, &dev->napi_list, dev_list) {
		if (napi->poll_owner == smp_processor_id())
			return 1;
	}
	return 0;
}

328
/* call with IRQ disabled */
329 330
void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
			     struct net_device *dev)
L
Linus Torvalds 已提交
331
{
S
Stephen Hemminger 已提交
332 333
	int status = NETDEV_TX_BUSY;
	unsigned long tries;
H
Herbert Xu 已提交
334
	/* It is up to the caller to keep npinfo alive. */
335
	struct netpoll_info *npinfo;
S
Stephen Hemminger 已提交
336

337 338 339
	WARN_ON_ONCE(!irqs_disabled());

	npinfo = rcu_dereference_bh(np->dev->npinfo);
340
	if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
341
		dev_kfree_skb_irq(skb);
342 343
		return;
	}
S
Stephen Hemminger 已提交
344 345

	/* don't get messages out of order, and no recursion */
346
	if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
347
		struct netdev_queue *txq;
348

349
		txq = netdev_pick_tx(dev, skb, NULL);
350

351 352 353
		/* try until next clock tick */
		for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
		     tries > 0; --tries) {
354
			if (HARD_TX_TRYLOCK(dev, txq)) {
355 356 357
				if (!netif_xmit_stopped(txq))
					status = netpoll_start_xmit(skb, dev, txq);

358
				HARD_TX_UNLOCK(dev, txq);
359 360 361 362 363

				if (status == NETDEV_TX_OK)
					break;

			}
364 365

			/* tickle device maybe there is some cleanup */
366
			netpoll_poll_dev(np->dev);
367 368

			udelay(USEC_PER_POLL);
M
Matt Mackall 已提交
369
		}
370 371

		WARN_ONCE(!irqs_disabled(),
372
			"netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pF)\n",
373
			dev->name, dev->netdev_ops->ndo_start_xmit);
374

L
Linus Torvalds 已提交
375 376
	}

S
Stephen Hemminger 已提交
377
	if (status != NETDEV_TX_OK) {
S
Stephen Hemminger 已提交
378
		skb_queue_tail(&npinfo->txq, skb);
379
		schedule_delayed_work(&npinfo->tx_work,0);
L
Linus Torvalds 已提交
380 381
	}
}
382
EXPORT_SYMBOL(netpoll_send_skb_on_dev);
L
Linus Torvalds 已提交
383 384 385

void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
{
386
	int total_len, ip_len, udp_len;
L
Linus Torvalds 已提交
387 388 389 390
	struct sk_buff *skb;
	struct udphdr *udph;
	struct iphdr *iph;
	struct ethhdr *eth;
391
	static atomic_t ip_ident;
C
Cong Wang 已提交
392
	struct ipv6hdr *ip6h;
L
Linus Torvalds 已提交
393

394 395
	WARN_ON_ONCE(!irqs_disabled());

L
Linus Torvalds 已提交
396
	udp_len = len + sizeof(*udph);
C
Cong Wang 已提交
397 398 399
	if (np->ipv6)
		ip_len = udp_len + sizeof(*ip6h);
	else
C
Cong Wang 已提交
400 401
		ip_len = udp_len + sizeof(*iph);

402
	total_len = ip_len + LL_RESERVED_SPACE(np->dev);
L
Linus Torvalds 已提交
403

404 405
	skb = find_skb(np, total_len + np->dev->needed_tailroom,
		       total_len - len);
L
Linus Torvalds 已提交
406 407 408
	if (!skb)
		return;

409
	skb_copy_to_linear_data(skb, msg, len);
410
	skb_put(skb, len);
L
Linus Torvalds 已提交
411

412 413 414
	skb_push(skb, sizeof(*udph));
	skb_reset_transport_header(skb);
	udph = udp_hdr(skb);
L
Linus Torvalds 已提交
415 416 417
	udph->source = htons(np->local_port);
	udph->dest = htons(np->remote_port);
	udph->len = htons(udp_len);
C
Cong Wang 已提交
418

C
Cong Wang 已提交
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
	if (np->ipv6) {
		udph->check = 0;
		udph->check = csum_ipv6_magic(&np->local_ip.in6,
					      &np->remote_ip.in6,
					      udp_len, IPPROTO_UDP,
					      csum_partial(udph, udp_len, 0));
		if (udph->check == 0)
			udph->check = CSUM_MANGLED_0;

		skb_push(skb, sizeof(*ip6h));
		skb_reset_network_header(skb);
		ip6h = ipv6_hdr(skb);

		/* ip6h->version = 6; ip6h->priority = 0; */
		put_unaligned(0x60, (unsigned char *)ip6h);
		ip6h->flow_lbl[0] = 0;
		ip6h->flow_lbl[1] = 0;
		ip6h->flow_lbl[2] = 0;

		ip6h->payload_len = htons(sizeof(struct udphdr) + len);
		ip6h->nexthdr = IPPROTO_UDP;
		ip6h->hop_limit = 32;
		ip6h->saddr = np->local_ip.in6;
		ip6h->daddr = np->remote_ip.in6;

444
		eth = skb_push(skb, ETH_HLEN);
C
Cong Wang 已提交
445 446 447
		skb_reset_mac_header(skb);
		skb->protocol = eth->h_proto = htons(ETH_P_IPV6);
	} else {
C
Cong Wang 已提交
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
		udph->check = 0;
		udph->check = csum_tcpudp_magic(np->local_ip.ip,
						np->remote_ip.ip,
						udp_len, IPPROTO_UDP,
						csum_partial(udph, udp_len, 0));
		if (udph->check == 0)
			udph->check = CSUM_MANGLED_0;

		skb_push(skb, sizeof(*iph));
		skb_reset_network_header(skb);
		iph = ip_hdr(skb);

		/* iph->version = 4; iph->ihl = 5; */
		put_unaligned(0x45, (unsigned char *)iph);
		iph->tos      = 0;
		put_unaligned(htons(ip_len), &(iph->tot_len));
		iph->id       = htons(atomic_inc_return(&ip_ident));
		iph->frag_off = 0;
		iph->ttl      = 64;
		iph->protocol = IPPROTO_UDP;
		iph->check    = 0;
		put_unaligned(np->local_ip.ip, &(iph->saddr));
		put_unaligned(np->remote_ip.ip, &(iph->daddr));
		iph->check    = ip_fast_csum((unsigned char *)iph, iph->ihl);

473
		eth = skb_push(skb, ETH_HLEN);
C
Cong Wang 已提交
474 475 476 477
		skb_reset_mac_header(skb);
		skb->protocol = eth->h_proto = htons(ETH_P_IP);
	}

J
Joe Perches 已提交
478 479
	ether_addr_copy(eth->h_source, np->dev->dev_addr);
	ether_addr_copy(eth->h_dest, np->remote_mac);
L
Linus Torvalds 已提交
480 481 482 483 484

	skb->dev = np->dev;

	netpoll_send_skb(np, skb);
}
E
Eric Dumazet 已提交
485
EXPORT_SYMBOL(netpoll_send_udp);
L
Linus Torvalds 已提交
486

487 488
void netpoll_print_options(struct netpoll *np)
{
489
	np_info(np, "local port %d\n", np->local_port);
C
Cong Wang 已提交
490 491 492
	if (np->ipv6)
		np_info(np, "local IPv6 address %pI6c\n", &np->local_ip.in6);
	else
C
Cong Wang 已提交
493
		np_info(np, "local IPv4 address %pI4\n", &np->local_ip.ip);
494 495
	np_info(np, "interface '%s'\n", np->dev_name);
	np_info(np, "remote port %d\n", np->remote_port);
C
Cong Wang 已提交
496 497 498
	if (np->ipv6)
		np_info(np, "remote IPv6 address %pI6c\n", &np->remote_ip.in6);
	else
C
Cong Wang 已提交
499
		np_info(np, "remote IPv4 address %pI4\n", &np->remote_ip.ip);
500
	np_info(np, "remote ethernet address %pM\n", np->remote_mac);
501
}
E
Eric Dumazet 已提交
502
EXPORT_SYMBOL(netpoll_print_options);
503

C
Cong Wang 已提交
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
{
	const char *end;

	if (!strchr(str, ':') &&
	    in4_pton(str, -1, (void *)addr, -1, &end) > 0) {
		if (!*end)
			return 0;
	}
	if (in6_pton(str, -1, addr->in6.s6_addr, -1, &end) > 0) {
#if IS_ENABLED(CONFIG_IPV6)
		if (!*end)
			return 1;
#else
		return -1;
#endif
	}
	return -1;
}

L
Linus Torvalds 已提交
524 525 526
int netpoll_parse_options(struct netpoll *np, char *opt)
{
	char *cur=opt, *delim;
C
Cong Wang 已提交
527
	int ipv6;
528
	bool ipversion_set = false;
L
Linus Torvalds 已提交
529

530
	if (*cur != '@') {
L
Linus Torvalds 已提交
531 532
		if ((delim = strchr(cur, '@')) == NULL)
			goto parse_failed;
533
		*delim = 0;
534 535
		if (kstrtou16(cur, 10, &np->local_port))
			goto parse_failed;
536
		cur = delim;
L
Linus Torvalds 已提交
537 538 539
	}
	cur++;

540
	if (*cur != '/') {
541
		ipversion_set = true;
L
Linus Torvalds 已提交
542 543
		if ((delim = strchr(cur, '/')) == NULL)
			goto parse_failed;
544
		*delim = 0;
C
Cong Wang 已提交
545 546 547 548 549
		ipv6 = netpoll_parse_ip_addr(cur, &np->local_ip);
		if (ipv6 < 0)
			goto parse_failed;
		else
			np->ipv6 = (bool)ipv6;
550
		cur = delim;
L
Linus Torvalds 已提交
551 552 553
	}
	cur++;

554
	if (*cur != ',') {
L
Linus Torvalds 已提交
555 556 557
		/* parse out dev name */
		if ((delim = strchr(cur, ',')) == NULL)
			goto parse_failed;
558
		*delim = 0;
L
Linus Torvalds 已提交
559
		strlcpy(np->dev_name, cur, sizeof(np->dev_name));
560
		cur = delim;
L
Linus Torvalds 已提交
561 562 563
	}
	cur++;

564
	if (*cur != '@') {
L
Linus Torvalds 已提交
565 566 567
		/* dst port */
		if ((delim = strchr(cur, '@')) == NULL)
			goto parse_failed;
568
		*delim = 0;
569
		if (*cur == ' ' || *cur == '\t')
570
			np_info(np, "warning: whitespace is not allowed\n");
571 572
		if (kstrtou16(cur, 10, &np->remote_port))
			goto parse_failed;
573
		cur = delim;
L
Linus Torvalds 已提交
574 575 576 577 578 579
	}
	cur++;

	/* dst ip */
	if ((delim = strchr(cur, '/')) == NULL)
		goto parse_failed;
580
	*delim = 0;
C
Cong Wang 已提交
581 582 583
	ipv6 = netpoll_parse_ip_addr(cur, &np->remote_ip);
	if (ipv6 < 0)
		goto parse_failed;
584
	else if (ipversion_set && np->ipv6 != (bool)ipv6)
C
Cong Wang 已提交
585 586 587
		goto parse_failed;
	else
		np->ipv6 = (bool)ipv6;
588
	cur = delim + 1;
L
Linus Torvalds 已提交
589

590
	if (*cur != 0) {
L
Linus Torvalds 已提交
591
		/* MAC address */
592
		if (!mac_pton(cur, np->remote_mac))
L
Linus Torvalds 已提交
593 594 595
			goto parse_failed;
	}

596
	netpoll_print_options(np);
L
Linus Torvalds 已提交
597 598 599 600

	return 0;

 parse_failed:
601
	np_info(np, "couldn't parse config at '%s'!\n", cur);
L
Linus Torvalds 已提交
602 603
	return -1;
}
E
Eric Dumazet 已提交
604
EXPORT_SYMBOL(netpoll_parse_options);
L
Linus Torvalds 已提交
605

606
int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
L
Linus Torvalds 已提交
607
{
608
	struct netpoll_info *npinfo;
H
Herbert Xu 已提交
609
	const struct net_device_ops *ops;
S
Stephen Hemminger 已提交
610
	int err;
L
Linus Torvalds 已提交
611

612
	np->dev = ndev;
613
	strlcpy(np->dev_name, ndev->name, IFNAMSIZ);
614
	INIT_WORK(&np->cleanup_work, netpoll_async_cleanup);
615

616 617
	if ((ndev->priv_flags & IFF_DISABLE_NETPOLL) ||
	    !ndev->netdev_ops->ndo_poll_controller) {
618 619
		np_err(np, "%s doesn't support polling, aborting\n",
		       np->dev_name);
620 621 622 623 624
		err = -ENOTSUPP;
		goto out;
	}

	if (!ndev->npinfo) {
625
		npinfo = kmalloc(sizeof(*npinfo), GFP_KERNEL);
626 627 628 629 630
		if (!npinfo) {
			err = -ENOMEM;
			goto out;
		}

631
		sema_init(&npinfo->dev_lock, 1);
632 633 634
		skb_queue_head_init(&npinfo->txq);
		INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);

635
		refcount_set(&npinfo->refcnt, 1);
636 637 638

		ops = np->dev->netdev_ops;
		if (ops->ndo_netpoll_setup) {
639
			err = ops->ndo_netpoll_setup(ndev, npinfo);
640 641 642 643
			if (err)
				goto free_npinfo;
		}
	} else {
N
Neil Horman 已提交
644
		npinfo = rtnl_dereference(ndev->npinfo);
645
		refcount_inc(&npinfo->refcnt);
646 647 648 649 650
	}

	npinfo->netpoll = np;

	/* last thing to do is link it to the net device structure */
651
	rcu_assign_pointer(ndev->npinfo, npinfo);
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667

	return 0;

free_npinfo:
	kfree(npinfo);
out:
	return err;
}
EXPORT_SYMBOL_GPL(__netpoll_setup);

int netpoll_setup(struct netpoll *np)
{
	struct net_device *ndev = NULL;
	struct in_device *in_dev;
	int err;

668
	rtnl_lock();
669 670 671 672
	if (np->dev_name) {
		struct net *net = current->nsproxy->net_ns;
		ndev = __dev_get_by_name(net, np->dev_name);
	}
L
Linus Torvalds 已提交
673
	if (!ndev) {
674
		np_err(np, "%s doesn't exist, aborting\n", np->dev_name);
675 676
		err = -ENODEV;
		goto unlock;
L
Linus Torvalds 已提交
677
	}
678
	dev_hold(ndev);
L
Linus Torvalds 已提交
679

680
	if (netdev_master_upper_dev_get(ndev)) {
681
		np_err(np, "%s is a slave device, aborting\n", np->dev_name);
682 683
		err = -EBUSY;
		goto put;
684 685
	}

L
Linus Torvalds 已提交
686 687 688
	if (!netif_running(ndev)) {
		unsigned long atmost, atleast;

689
		np_info(np, "device %s not up yet, forcing it\n", np->dev_name);
L
Linus Torvalds 已提交
690

S
Stephen Hemminger 已提交
691 692 693
		err = dev_open(ndev);

		if (err) {
694
			np_err(np, "failed to open %s\n", ndev->name);
695
			goto put;
L
Linus Torvalds 已提交
696 697
		}

698
		rtnl_unlock();
L
Linus Torvalds 已提交
699
		atleast = jiffies + HZ/10;
700
		atmost = jiffies + carrier_timeout * HZ;
L
Linus Torvalds 已提交
701 702
		while (!netif_carrier_ok(ndev)) {
			if (time_after(jiffies, atmost)) {
703
				np_notice(np, "timeout waiting for carrier\n");
L
Linus Torvalds 已提交
704 705
				break;
			}
706
			msleep(1);
L
Linus Torvalds 已提交
707 708 709 710 711 712 713 714
		}

		/* If carrier appears to come up instantly, we don't
		 * trust it and pause so that we don't pump all our
		 * queued console messages into the bitbucket.
		 */

		if (time_before(jiffies, atleast)) {
715
			np_notice(np, "carrier detect appears untrustworthy, waiting 4 seconds\n");
L
Linus Torvalds 已提交
716 717
			msleep(4000);
		}
718
		rtnl_lock();
L
Linus Torvalds 已提交
719 720
	}

C
Cong Wang 已提交
721 722
	if (!np->local_ip.ip) {
		if (!np->ipv6) {
723
			in_dev = __in_dev_get_rtnl(ndev);
C
Cong Wang 已提交
724 725 726 727 728 729 730 731 732 733

			if (!in_dev || !in_dev->ifa_list) {
				np_err(np, "no IP address for %s, aborting\n",
				       np->dev_name);
				err = -EDESTADDRREQ;
				goto put;
			}

			np->local_ip.ip = in_dev->ifa_list->ifa_local;
			np_info(np, "local IP %pI4\n", &np->local_ip.ip);
C
Cong Wang 已提交
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
		} else {
#if IS_ENABLED(CONFIG_IPV6)
			struct inet6_dev *idev;

			err = -EDESTADDRREQ;
			idev = __in6_dev_get(ndev);
			if (idev) {
				struct inet6_ifaddr *ifp;

				read_lock_bh(&idev->lock);
				list_for_each_entry(ifp, &idev->addr_list, if_list) {
					if (ipv6_addr_type(&ifp->addr) & IPV6_ADDR_LINKLOCAL)
						continue;
					np->local_ip.in6 = ifp->addr;
					err = 0;
					break;
				}
				read_unlock_bh(&idev->lock);
			}
			if (err) {
				np_err(np, "no IPv6 address for %s, aborting\n",
				       np->dev_name);
				goto put;
			} else
				np_info(np, "local IPv6 %pI6c\n", &np->local_ip.in6);
#else
			np_err(np, "IPv6 is not supported %s, aborting\n",
			       np->dev_name);
762
			err = -EINVAL;
C
Cong Wang 已提交
763 764
			goto put;
#endif
L
Linus Torvalds 已提交
765 766 767
		}
	}

768 769 770
	/* fill up the skb queue */
	refill_skbs();

771
	err = __netpoll_setup(np, ndev);
772 773 774
	if (err)
		goto put;

775
	rtnl_unlock();
L
Linus Torvalds 已提交
776 777
	return 0;

778
put:
L
Linus Torvalds 已提交
779
	dev_put(ndev);
780 781
unlock:
	rtnl_unlock();
S
Stephen Hemminger 已提交
782
	return err;
L
Linus Torvalds 已提交
783
}
E
Eric Dumazet 已提交
784
EXPORT_SYMBOL(netpoll_setup);
L
Linus Torvalds 已提交
785

786 787
static int __init netpoll_init(void)
{
788 789 790 791 792
	skb_queue_head_init(&skb_pool);
	return 0;
}
core_initcall(netpoll_init);

793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
{
	struct netpoll_info *npinfo =
			container_of(rcu_head, struct netpoll_info, rcu);

	skb_queue_purge(&npinfo->txq);

	/* we can't call cancel_delayed_work_sync here, as we are in softirq */
	cancel_delayed_work(&npinfo->tx_work);

	/* clean after last, unfinished work */
	__skb_queue_purge(&npinfo->txq);
	/* now cancel it again */
	cancel_delayed_work(&npinfo->tx_work);
	kfree(npinfo);
}

810
void __netpoll_cleanup(struct netpoll *np)
L
Linus Torvalds 已提交
811
{
812 813
	struct netpoll_info *npinfo;

N
Neil Horman 已提交
814 815 816 817 818
	/* rtnl_dereference would be preferable here but
	 * rcu_cleanup_netpoll path can put us in here safely without
	 * holding the rtnl, so plain rcu_dereference it is
	 */
	npinfo = rtnl_dereference(np->dev->npinfo);
819
	if (!npinfo)
820
		return;
S
Stephen Hemminger 已提交
821

822 823
	synchronize_srcu(&netpoll_srcu);

824
	if (refcount_dec_and_test(&npinfo->refcnt)) {
825
		const struct net_device_ops *ops;
H
Herbert Xu 已提交
826

827 828 829
		ops = np->dev->netdev_ops;
		if (ops->ndo_netpoll_cleanup)
			ops->ndo_netpoll_cleanup(np->dev);
H
Herbert Xu 已提交
830

831
		RCU_INIT_POINTER(np->dev->npinfo, NULL);
832
		call_rcu_bh(&npinfo->rcu, rcu_cleanup_netpoll_info);
D
david decotigny 已提交
833 834
	} else
		RCU_INIT_POINTER(np->dev->npinfo, NULL);
835 836
}
EXPORT_SYMBOL_GPL(__netpoll_cleanup);
H
Herbert Xu 已提交
837

838
static void netpoll_async_cleanup(struct work_struct *work)
839
{
840
	struct netpoll *np = container_of(work, struct netpoll, cleanup_work);
S
Stephen Hemminger 已提交
841

842
	rtnl_lock();
843
	__netpoll_cleanup(np);
844
	rtnl_unlock();
845 846
	kfree(np);
}
S
Stephen Hemminger 已提交
847

848
void __netpoll_free_async(struct netpoll *np)
849
{
850
	schedule_work(&np->cleanup_work);
851
}
852
EXPORT_SYMBOL_GPL(__netpoll_free_async);
853

854 855 856
void netpoll_cleanup(struct netpoll *np)
{
	rtnl_lock();
857 858
	if (!np->dev)
		goto out;
859 860
	__netpoll_cleanup(np);
	dev_put(np->dev);
L
Linus Torvalds 已提交
861
	np->dev = NULL;
862 863
out:
	rtnl_unlock();
L
Linus Torvalds 已提交
864
}
E
Eric Dumazet 已提交
865
EXPORT_SYMBOL(netpoll_cleanup);