virtio_net.c 84.8 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
2
/* A network driver using virtio.
R
Rusty Russell 已提交
3 4 5 6 7 8
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
9
#include <linux/ethtool.h>
R
Rusty Russell 已提交
10 11 12
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
J
John Fastabend 已提交
13
#include <linux/bpf.h>
14
#include <linux/bpf_trace.h>
R
Rusty Russell 已提交
15
#include <linux/scatterlist.h>
16
#include <linux/if_vlan.h>
17
#include <linux/slab.h>
18
#include <linux/cpu.h>
19
#include <linux/average.h>
J
Jason Wang 已提交
20
#include <linux/filter.h>
21
#include <linux/kernel.h>
22
#include <net/route.h>
23
#include <net/xdp.h>
24
#include <net/net_failover.h>
R
Rusty Russell 已提交
25

26
static int napi_weight = NAPI_POLL_WEIGHT;
27 28
module_param(napi_weight, int, 0444);

29
static bool csum = true, gso = true, napi_tx = true;
R
Rusty Russell 已提交
30 31
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);
W
Willem de Bruijn 已提交
32
module_param(napi_tx, bool, 0644);
R
Rusty Russell 已提交
33

R
Rusty Russell 已提交
34
/* FIXME: MTU in config. */
35
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
36
#define GOOD_COPY_LEN	128
R
Rusty Russell 已提交
37

38 39
#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)

40 41 42
/* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
#define VIRTIO_XDP_HEADROOM 256

43 44 45 46
/* Separating two types of XDP xmit */
#define VIRTIO_XDP_TX		BIT(0)
#define VIRTIO_XDP_REDIR	BIT(1)

47 48
#define VIRTIO_XDP_FLAG	BIT(0)

J
Johannes Berg 已提交
49 50 51 52
/* RX packet size EWMA. The average packet size is used to determine the packet
 * buffer size when refilling RX rings. As the entire RX ring may be refilled
 * at once, the weight is chosen so that the EWMA will be insensitive to short-
 * term, transient changes in packet size.
53
 */
54
DECLARE_EWMA(pkt_len, 0, 64)
55

56
#define VIRTNET_DRIVER_VERSION "1.0.0"
57

58 59 60 61
static const unsigned long guest_offloads[] = {
	VIRTIO_NET_F_GUEST_TSO4,
	VIRTIO_NET_F_GUEST_TSO6,
	VIRTIO_NET_F_GUEST_ECN,
62 63
	VIRTIO_NET_F_GUEST_UFO,
	VIRTIO_NET_F_GUEST_CSUM
64
};
65

T
Toshiaki Makita 已提交
66 67 68
struct virtnet_stat_desc {
	char desc[ETH_GSTRING_LEN];
	size_t offset;
69 70
};

T
Toshiaki Makita 已提交
71 72 73 74
struct virtnet_sq_stats {
	struct u64_stats_sync syncp;
	u64 packets;
	u64 bytes;
75 76
	u64 xdp_tx;
	u64 xdp_tx_drops;
T
Toshiaki Makita 已提交
77
	u64 kicks;
T
Toshiaki Makita 已提交
78 79
};

80 81
struct virtnet_rq_stats {
	struct u64_stats_sync syncp;
T
Toshiaki Makita 已提交
82 83
	u64 packets;
	u64 bytes;
84
	u64 drops;
85 86 87 88
	u64 xdp_packets;
	u64 xdp_tx;
	u64 xdp_redirects;
	u64 xdp_drops;
T
Toshiaki Makita 已提交
89
	u64 kicks;
T
Toshiaki Makita 已提交
90 91 92
};

#define VIRTNET_SQ_STAT(m)	offsetof(struct virtnet_sq_stats, m)
93
#define VIRTNET_RQ_STAT(m)	offsetof(struct virtnet_rq_stats, m)
T
Toshiaki Makita 已提交
94 95

static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
96 97 98 99
	{ "packets",		VIRTNET_SQ_STAT(packets) },
	{ "bytes",		VIRTNET_SQ_STAT(bytes) },
	{ "xdp_tx",		VIRTNET_SQ_STAT(xdp_tx) },
	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
T
Toshiaki Makita 已提交
100
	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
T
Toshiaki Makita 已提交
101 102 103
};

static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
104 105 106 107 108 109 110
	{ "packets",		VIRTNET_RQ_STAT(packets) },
	{ "bytes",		VIRTNET_RQ_STAT(bytes) },
	{ "drops",		VIRTNET_RQ_STAT(drops) },
	{ "xdp_packets",	VIRTNET_RQ_STAT(xdp_packets) },
	{ "xdp_tx",		VIRTNET_RQ_STAT(xdp_tx) },
	{ "xdp_redirects",	VIRTNET_RQ_STAT(xdp_redirects) },
	{ "xdp_drops",		VIRTNET_RQ_STAT(xdp_drops) },
T
Toshiaki Makita 已提交
111
	{ "kicks",		VIRTNET_RQ_STAT(kicks) },
T
Toshiaki Makita 已提交
112 113 114 115 116
};

#define VIRTNET_SQ_STATS_LEN	ARRAY_SIZE(virtnet_sq_stats_desc)
#define VIRTNET_RQ_STATS_LEN	ARRAY_SIZE(virtnet_rq_stats_desc)

117 118 119 120 121 122 123
/* Internal representation of a send virtqueue */
struct send_queue {
	/* Virtqueue associated with this send _queue */
	struct virtqueue *vq;

	/* TX: fragments + linear part + virtio header */
	struct scatterlist sg[MAX_SKB_FRAGS + 2];
J
Jason Wang 已提交
124 125 126

	/* Name of the send queue: output.$index */
	char name[40];
W
Willem de Bruijn 已提交
127

T
Toshiaki Makita 已提交
128 129
	struct virtnet_sq_stats stats;

W
Willem de Bruijn 已提交
130
	struct napi_struct napi;
131 132 133 134 135 136 137
};

/* Internal representation of a receive virtqueue */
struct receive_queue {
	/* Virtqueue associated with this receive_queue */
	struct virtqueue *vq;

R
Rusty Russell 已提交
138 139
	struct napi_struct napi;

J
John Fastabend 已提交
140 141
	struct bpf_prog __rcu *xdp_prog;

T
Toshiaki Makita 已提交
142 143
	struct virtnet_rq_stats stats;

144 145 146
	/* Chain pages by the private ptr. */
	struct page *pages;

147
	/* Average packet length for mergeable receive buffers. */
J
Johannes Berg 已提交
148
	struct ewma_pkt_len mrg_avg_pkt_len;
149

150 151 152
	/* Page frag for packet buffer allocation. */
	struct page_frag alloc_frag;

153 154
	/* RX: fragments + linear part + virtio header */
	struct scatterlist sg[MAX_SKB_FRAGS + 2];
J
Jason Wang 已提交
155

156 157 158
	/* Min single buffer size for mergeable buffers case. */
	unsigned int min_buf_len;

J
Jason Wang 已提交
159 160
	/* Name of this receive queue: input.$index */
	char name[40];
161 162

	struct xdp_rxq_info xdp_rxq;
163 164
};

165 166 167 168 169 170 171
/* Control VQ buffers: protected by the rtnl lock */
struct control_buf {
	struct virtio_net_ctrl_hdr hdr;
	virtio_net_ctrl_ack status;
	struct virtio_net_ctrl_mq mq;
	u8 promisc;
	u8 allmulti;
172
	__virtio16 vid;
173
	__virtio64 offloads;
174 175
};

176 177 178 179
struct virtnet_info {
	struct virtio_device *vdev;
	struct virtqueue *cvq;
	struct net_device *dev;
J
Jason Wang 已提交
180 181
	struct send_queue *sq;
	struct receive_queue *rq;
182 183
	unsigned int status;

J
Jason Wang 已提交
184 185 186 187 188 189
	/* Max # of queue pairs supported by the device */
	u16 max_queue_pairs;

	/* # of queue pairs currently used by the driver */
	u16 curr_queue_pairs;

190 191 192
	/* # of XDP queue pairs currently used by the driver */
	u16 xdp_queue_pairs;

193 194 195
	/* I like... big packets and I cannot lie! */
	bool big_packets;

196 197 198
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

J
Jason Wang 已提交
199 200 201
	/* Has control virtqueue */
	bool has_cvq;

202 203 204
	/* Host can handle any s/g split between our header and packet data */
	bool any_header_sg;

205 206 207
	/* Packet virtio header size */
	u8 hdr_len;

208 209 210
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

211 212 213
	/* Work struct for config space updates */
	struct work_struct config_work;

J
Jason Wang 已提交
214 215
	/* Does the affinity hint is set for virtqueues? */
	bool affinity_hint_set;
216

217 218 219
	/* CPU hotplug instances for online & dead */
	struct hlist_node node;
	struct hlist_node node_dead;
220

221
	struct control_buf *ctrl;
222 223 224 225

	/* Ethtool settings */
	u8 duplex;
	u32 speed;
226 227

	unsigned long guest_offloads;
228
	unsigned long guest_offloads_capable;
229 230 231

	/* failover when STANDBY feature enabled */
	struct failover *failover;
R
Rusty Russell 已提交
232 233
};

234
struct padded_vnet_hdr {
235
	struct virtio_net_hdr_mrg_rxbuf hdr;
236
	/*
237 238 239
	 * hdr is in a separate sg buffer, and data sg buffer shares same page
	 * with this header sg. This padding makes next sg 16 byte aligned
	 * after the header.
240
	 */
241
	char padding[4];
242 243
};

244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
static bool is_xdp_frame(void *ptr)
{
	return (unsigned long)ptr & VIRTIO_XDP_FLAG;
}

static void *xdp_to_ptr(struct xdp_frame *ptr)
{
	return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
}

static struct xdp_frame *ptr_to_xdp(void *ptr)
{
	return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}

J
Jason Wang 已提交
259 260 261 262 263
/* Converting between virtqueue no. and kernel tx/rx queue no.
 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 */
static int vq2txq(struct virtqueue *vq)
{
264
	return (vq->index - 1) / 2;
J
Jason Wang 已提交
265 266 267 268 269 270 271 272 273
}

static int txq2vq(int txq)
{
	return txq * 2 + 1;
}

static int vq2rxq(struct virtqueue *vq)
{
274
	return vq->index / 2;
J
Jason Wang 已提交
275 276 277 278 279 280 281
}

static int rxq2vq(int rxq)
{
	return rxq * 2;
}

282
static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
R
Rusty Russell 已提交
283
{
284
	return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
R
Rusty Russell 已提交
285 286
}

287 288 289 290
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
291
static void give_pages(struct receive_queue *rq, struct page *page)
292
{
293
	struct page *end;
294

295
	/* Find end of list, sew whole thing into vi->rq.pages. */
296
	for (end = page; end->private; end = (struct page *)end->private);
297 298
	end->private = (unsigned long)rq->pages;
	rq->pages = page;
299 300
}

301
static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
302
{
303
	struct page *p = rq->pages;
304

305
	if (p) {
306
		rq->pages = (struct page *)p->private;
307 308 309
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
310 311 312 313
		p = alloc_page(gfp_mask);
	return p;
}

314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
static void virtqueue_napi_schedule(struct napi_struct *napi,
				    struct virtqueue *vq)
{
	if (napi_schedule_prep(napi)) {
		virtqueue_disable_cb(vq);
		__napi_schedule(napi);
	}
}

static void virtqueue_napi_complete(struct napi_struct *napi,
				    struct virtqueue *vq, int processed)
{
	int opaque;

	opaque = virtqueue_enable_cb_prepare(vq);
329 330 331 332 333 334
	if (napi_complete_done(napi, processed)) {
		if (unlikely(virtqueue_poll(vq, opaque)))
			virtqueue_napi_schedule(napi, vq);
	} else {
		virtqueue_disable_cb(vq);
	}
335 336
}

337
static void skb_xmit_done(struct virtqueue *vq)
R
Rusty Russell 已提交
338
{
339
	struct virtnet_info *vi = vq->vdev->priv;
W
Willem de Bruijn 已提交
340
	struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
R
Rusty Russell 已提交
341

342
	/* Suppress further interrupts. */
343
	virtqueue_disable_cb(vq);
344

W
Willem de Bruijn 已提交
345 346 347 348 349
	if (napi->weight)
		virtqueue_napi_schedule(napi, vq);
	else
		/* We were probably waiting for more output buffers. */
		netif_wake_subqueue(vi->dev, vq2txq(vq));
R
Rusty Russell 已提交
350 351
}

352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
#define MRG_CTX_HEADER_SHIFT 22
static void *mergeable_len_to_ctx(unsigned int truesize,
				  unsigned int headroom)
{
	return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
}

static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
{
	return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
}

static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
{
	return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
}

369
/* Called from bottom half context */
M
Michael S. Tsirkin 已提交
370 371
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct receive_queue *rq,
372
				   struct page *page, unsigned int offset,
373 374
				   unsigned int len, unsigned int truesize,
				   bool hdr_valid)
375 376
{
	struct sk_buff *skb;
377
	struct virtio_net_hdr_mrg_rxbuf *hdr;
378
	unsigned int copy, hdr_len, hdr_padded_len;
379
	char *p;
380

381
	p = page_address(page) + offset;
382

383
	/* copy small packet so we can reuse these pages for small data */
384
	skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
385 386
	if (unlikely(!skb))
		return NULL;
387

388
	hdr = skb_vnet_hdr(skb);
389

390 391
	hdr_len = vi->hdr_len;
	if (vi->mergeable_rx_bufs)
392
		hdr_padded_len = sizeof(*hdr);
393
	else
394
		hdr_padded_len = sizeof(struct padded_vnet_hdr);
395

396 397
	if (hdr_valid)
		memcpy(hdr, p, hdr_len);
398

399
	len -= hdr_len;
400 401
	offset += hdr_padded_len;
	p += hdr_padded_len;
402

403 404 405
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
406
	skb_put_data(skb, p, copy);
407

408 409
	len -= copy;
	offset += copy;
410

411 412 413 414 415 416 417 418
	if (vi->mergeable_rx_bufs) {
		if (len)
			skb_add_rx_frag(skb, 0, page, offset, len, truesize);
		else
			put_page(page);
		return skb;
	}

419 420 421 422 423 424 425
	/*
	 * Verify that we can indeed put this data into a skb.
	 * This is here to handle cases when the device erroneously
	 * tries to receive more than is possible. This is usually
	 * the case of a broken device.
	 */
	if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
426
		net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
427 428 429
		dev_kfree_skb(skb);
		return NULL;
	}
430
	BUG_ON(offset >= PAGE_SIZE);
431
	while (len) {
432 433 434 435
		unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
				frag_size, truesize);
		len -= frag_size;
436 437 438
		page = (struct page *)page->private;
		offset = 0;
	}
439

440
	if (page)
441
		give_pages(rq, page);
442

443 444
	return skb;
}
445

446 447 448
static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
				   struct send_queue *sq,
				   struct xdp_frame *xdpf)
J
John Fastabend 已提交
449 450 451 452
{
	struct virtio_net_hdr_mrg_rxbuf *hdr;
	int err;

453 454 455
	/* virtqueue want to use data area in-front of packet */
	if (unlikely(xdpf->metasize > 0))
		return -EOPNOTSUPP;
J
John Fastabend 已提交
456

457 458 459 460 461
	if (unlikely(xdpf->headroom < vi->hdr_len))
		return -EOVERFLOW;

	/* Make room for virtqueue hdr (also change xdpf->headroom?) */
	xdpf->data -= vi->hdr_len;
462
	/* Zero header and leave csum up to XDP layers */
463
	hdr = xdpf->data;
464
	memset(hdr, 0, vi->hdr_len);
465
	xdpf->len   += vi->hdr_len;
466

467
	sg_init_one(sq->sg, xdpf->data, xdpf->len);
468

469 470
	err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf),
				   GFP_ATOMIC);
471
	if (unlikely(err))
472
		return -ENOSPC; /* Caller handle free/refcnt */
J
John Fastabend 已提交
473

474
	return 0;
J
John Fastabend 已提交
475 476
}

477 478 479 480 481 482 483 484
static struct send_queue *virtnet_xdp_sq(struct virtnet_info *vi)
{
	unsigned int qp;

	qp = vi->curr_queue_pairs - vi->xdp_queue_pairs + smp_processor_id();
	return &vi->sq[qp];
}

485
static int virtnet_xdp_xmit(struct net_device *dev,
486
			    int n, struct xdp_frame **frames, u32 flags)
J
Jason Wang 已提交
487 488
{
	struct virtnet_info *vi = netdev_priv(dev);
489 490
	struct receive_queue *rq = vi->rq;
	struct bpf_prog *xdp_prog;
491 492
	struct send_queue *sq;
	unsigned int len;
493 494
	int packets = 0;
	int bytes = 0;
495
	int drops = 0;
T
Toshiaki Makita 已提交
496
	int kicks = 0;
497
	int ret, err;
498
	void *ptr;
499 500
	int i;

501 502 503 504
	/* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
	 * indicate XDP resources have been successfully allocated.
	 */
	xdp_prog = rcu_dereference(rq->xdp_prog);
505 506 507 508 509 510 511
	if (!xdp_prog)
		return -ENXIO;

	sq = virtnet_xdp_sq(vi);

	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
		ret = -EINVAL;
512 513 514
		drops = n;
		goto out;
	}
515

516
	/* Free up any pending old buffers before queueing new ones. */
517
	while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
518 519 520 521 522 523 524 525 526 527 528 529
		if (likely(is_xdp_frame(ptr))) {
			struct xdp_frame *frame = ptr_to_xdp(ptr);

			bytes += frame->len;
			xdp_return_frame(frame);
		} else {
			struct sk_buff *skb = ptr;

			bytes += skb->len;
			napi_consume_skb(skb, false);
		}
		packets++;
530
	}
531 532 533 534 535 536 537 538 539 540

	for (i = 0; i < n; i++) {
		struct xdp_frame *xdpf = frames[i];

		err = __virtnet_xdp_xmit_one(vi, sq, xdpf);
		if (err) {
			xdp_return_frame_rx_napi(xdpf);
			drops++;
		}
	}
541
	ret = n - drops;
542

T
Toshiaki Makita 已提交
543 544 545 546
	if (flags & XDP_XMIT_FLUSH) {
		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
			kicks = 1;
	}
547 548
out:
	u64_stats_update_begin(&sq->stats.syncp);
549 550
	sq->stats.bytes += bytes;
	sq->stats.packets += packets;
551 552
	sq->stats.xdp_tx += n;
	sq->stats.xdp_tx_drops += drops;
T
Toshiaki Makita 已提交
553
	sq->stats.kicks += kicks;
554
	u64_stats_update_end(&sq->stats.syncp);
555

556
	return ret;
J
Jason Wang 已提交
557 558
}

559 560 561 562 563
static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
{
	return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
}

564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
/* We copy the packet for XDP in the following cases:
 *
 * 1) Packet is scattered across multiple rx buffers.
 * 2) Headroom space is insufficient.
 *
 * This is inefficient but it's a temporary condition that
 * we hit right after XDP is enabled and until queue is refilled
 * with large buffers with sufficient headroom - so it should affect
 * at most queue size packets.
 * Afterwards, the conditions to enable
 * XDP should preclude the underlying device from sending packets
 * across multiple buffers (num_buf > 1), and we make sure buffers
 * have enough headroom.
 */
static struct page *xdp_linearize_page(struct receive_queue *rq,
				       u16 *num_buf,
				       struct page *p,
				       int offset,
				       int page_off,
				       unsigned int *len)
{
	struct page *page = alloc_page(GFP_ATOMIC);

	if (!page)
		return NULL;

	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
	page_off += *len;

	while (--*num_buf) {
594
		int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
595 596 597 598 599 600 601 602 603 604 605 606 607 608
		unsigned int buflen;
		void *buf;
		int off;

		buf = virtqueue_get_buf(rq->vq, &buflen);
		if (unlikely(!buf))
			goto err_buf;

		p = virt_to_head_page(buf);
		off = buf - page_address(p);

		/* guard against a misconfigured or uncooperative backend that
		 * is sending packet larger than the MTU.
		 */
609
		if ((page_off + buflen + tailroom) > PAGE_SIZE) {
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
			put_page(p);
			goto err_buf;
		}

		memcpy(page_address(page) + page_off,
		       page_address(p) + off, buflen);
		page_off += buflen;
		put_page(p);
	}

	/* Headroom does not contribute to packet length */
	*len = page_off - VIRTIO_XDP_HEADROOM;
	return page;
err_buf:
	__free_pages(page, 0);
	return NULL;
}

628 629 630
static struct sk_buff *receive_small(struct net_device *dev,
				     struct virtnet_info *vi,
				     struct receive_queue *rq,
631
				     void *buf, void *ctx,
J
Jason Wang 已提交
632
				     unsigned int len,
633
				     unsigned int *xdp_xmit,
634
				     struct virtnet_rq_stats *stats)
635
{
636
	struct sk_buff *skb;
637
	struct bpf_prog *xdp_prog;
638
	unsigned int xdp_headroom = (unsigned long)ctx;
639 640 641 642
	unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
	unsigned int headroom = vi->hdr_len + header_offset;
	unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
643
	struct page *page = virt_to_head_page(buf);
644
	unsigned int delta = 0;
645
	struct page *xdp_page;
646 647
	int err;

648
	len -= vi->hdr_len;
649
	stats->bytes += len;
650

651 652 653
	rcu_read_lock();
	xdp_prog = rcu_dereference(rq->xdp_prog);
	if (xdp_prog) {
654
		struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
655
		struct xdp_frame *xdpf;
656
		struct xdp_buff xdp;
657
		void *orig_data;
658 659
		u32 act;

660
		if (unlikely(hdr->hdr.gso_type))
661
			goto err_xdp;
662

663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
			int offset = buf - page_address(page) + header_offset;
			unsigned int tlen = len + vi->hdr_len;
			u16 num_buf = 1;

			xdp_headroom = virtnet_get_headroom(vi);
			header_offset = VIRTNET_RX_PAD + xdp_headroom;
			headroom = vi->hdr_len + header_offset;
			buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
			xdp_page = xdp_linearize_page(rq, &num_buf, page,
						      offset, header_offset,
						      &tlen);
			if (!xdp_page)
				goto err_xdp;

			buf = page_address(xdp_page);
			put_page(page);
			page = xdp_page;
		}

684 685
		xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
		xdp.data = xdp.data_hard_start + xdp_headroom;
686
		xdp_set_data_meta_invalid(&xdp);
687
		xdp.data_end = xdp.data + len;
688
		xdp.rxq = &rq->xdp_rxq;
689
		orig_data = xdp.data;
690
		act = bpf_prog_run_xdp(xdp_prog, &xdp);
691
		stats->xdp_packets++;
692

693 694
		switch (act) {
		case XDP_PASS:
695
			/* Recalculate length in case bpf program changed it */
696
			delta = orig_data - xdp.data;
697
			len = xdp.data_end - xdp.data;
698 699
			break;
		case XDP_TX:
700
			stats->xdp_tx++;
701 702 703
			xdpf = convert_to_xdp_frame(&xdp);
			if (unlikely(!xdpf))
				goto err_xdp;
704 705
			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
			if (unlikely(err < 0)) {
706
				trace_xdp_exception(vi->dev, xdp_prog, act);
707 708
				goto err_xdp;
			}
709
			*xdp_xmit |= VIRTIO_XDP_TX;
J
Jason Wang 已提交
710 711 712
			rcu_read_unlock();
			goto xdp_xmit;
		case XDP_REDIRECT:
713
			stats->xdp_redirects++;
J
Jason Wang 已提交
714
			err = xdp_do_redirect(dev, &xdp, xdp_prog);
715 716
			if (err)
				goto err_xdp;
717
			*xdp_xmit |= VIRTIO_XDP_REDIR;
718 719 720
			rcu_read_unlock();
			goto xdp_xmit;
		default:
721
			bpf_warn_invalid_xdp_action(act);
722
			/* fall through */
723 724 725
		case XDP_ABORTED:
			trace_xdp_exception(vi->dev, xdp_prog, act);
		case XDP_DROP:
726 727 728 729 730
			goto err_xdp;
		}
	}
	rcu_read_unlock();

731 732
	skb = build_skb(buf, buflen);
	if (!skb) {
733
		put_page(page);
734 735 736
		goto err;
	}
	skb_reserve(skb, headroom - delta);
737
	skb_put(skb, len);
738 739 740 741 742 743
	if (!delta) {
		buf += header_offset;
		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
	} /* keep zeroed vnet hdr since packet was changed by bpf */

err:
744
	return skb;
745 746 747

err_xdp:
	rcu_read_unlock();
748 749
	stats->xdp_drops++;
	stats->drops++;
750
	put_page(page);
751 752
xdp_xmit:
	return NULL;
753 754 755
}

static struct sk_buff *receive_big(struct net_device *dev,
M
Michael S. Tsirkin 已提交
756
				   struct virtnet_info *vi,
757 758
				   struct receive_queue *rq,
				   void *buf,
759
				   unsigned int len,
760
				   struct virtnet_rq_stats *stats)
761 762
{
	struct page *page = buf;
763 764
	struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len,
					  PAGE_SIZE, true);
J
John Fastabend 已提交
765

766
	stats->bytes += len - vi->hdr_len;
767 768 769 770 771 772
	if (unlikely(!skb))
		goto err;

	return skb;

err:
773
	stats->drops++;
774 775 776 777
	give_pages(rq, page);
	return NULL;
}

778
static struct sk_buff *receive_mergeable(struct net_device *dev,
M
Michael S. Tsirkin 已提交
779
					 struct virtnet_info *vi,
780
					 struct receive_queue *rq,
781 782
					 void *buf,
					 void *ctx,
J
Jason Wang 已提交
783
					 unsigned int len,
784
					 unsigned int *xdp_xmit,
785
					 struct virtnet_rq_stats *stats)
786
{
787 788
	struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
789 790
	struct page *page = virt_to_head_page(buf);
	int offset = buf - page_address(page);
J
John Fastabend 已提交
791 792 793
	struct sk_buff *head_skb, *curr_skb;
	struct bpf_prog *xdp_prog;
	unsigned int truesize;
794
	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
795
	int err;
J
John Fastabend 已提交
796

J
John Fastabend 已提交
797
	head_skb = NULL;
798
	stats->bytes += len - vi->hdr_len;
J
John Fastabend 已提交
799

J
John Fastabend 已提交
800 801 802
	rcu_read_lock();
	xdp_prog = rcu_dereference(rq->xdp_prog);
	if (xdp_prog) {
803
		struct xdp_frame *xdpf;
804
		struct page *xdp_page;
805 806
		struct xdp_buff xdp;
		void *data;
J
John Fastabend 已提交
807 808
		u32 act;

809 810 811 812 813 814 815
		/* Transient failure which in theory could occur if
		 * in-flight packets from before XDP was enabled reach
		 * the receive path after XDP is loaded.
		 */
		if (unlikely(hdr->hdr.gso_type))
			goto err_xdp;

816 817 818 819 820 821
		/* This happens when rx buffer size is underestimated
		 * or headroom is not enough because of the buffer
		 * was refilled before XDP is set. This should only
		 * happen for the first several packets, so we don't
		 * care much about its performance.
		 */
822 823
		if (unlikely(num_buf > 1 ||
			     headroom < virtnet_get_headroom(vi))) {
824
			/* linearize data for XDP */
825
			xdp_page = xdp_linearize_page(rq, &num_buf,
826 827 828
						      page, offset,
						      VIRTIO_XDP_HEADROOM,
						      &len);
829 830
			if (!xdp_page)
				goto err_xdp;
831
			offset = VIRTIO_XDP_HEADROOM;
832 833
		} else {
			xdp_page = page;
J
John Fastabend 已提交
834 835
		}

836 837 838
		/* Allow consuming headroom but reserve enough space to push
		 * the descriptor on if we get an XDP_TX return code.
		 */
839
		data = page_address(xdp_page) + offset;
840
		xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len;
841
		xdp.data = data + vi->hdr_len;
842
		xdp_set_data_meta_invalid(&xdp);
843
		xdp.data_end = xdp.data + (len - vi->hdr_len);
844 845
		xdp.rxq = &rq->xdp_rxq;

846
		act = bpf_prog_run_xdp(xdp_prog, &xdp);
847
		stats->xdp_packets++;
848

J
John Fastabend 已提交
849 850
		switch (act) {
		case XDP_PASS:
851 852 853 854 855 856 857
			/* recalculate offset to account for any header
			 * adjustments. Note other cases do not build an
			 * skb and avoid using offset
			 */
			offset = xdp.data -
					page_address(xdp_page) - vi->hdr_len;

858 859 860
			/* recalculate len if xdp.data or xdp.data_end were
			 * adjusted
			 */
861
			len = xdp.data_end - xdp.data + vi->hdr_len;
862 863 864 865 866
			/* We can only create skb based on xdp_page. */
			if (unlikely(xdp_page != page)) {
				rcu_read_unlock();
				put_page(page);
				head_skb = page_to_skb(vi, rq, xdp_page,
867 868
						       offset, len,
						       PAGE_SIZE, false);
869 870
				return head_skb;
			}
J
John Fastabend 已提交
871 872
			break;
		case XDP_TX:
873
			stats->xdp_tx++;
874 875 876
			xdpf = convert_to_xdp_frame(&xdp);
			if (unlikely(!xdpf))
				goto err_xdp;
877 878
			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
			if (unlikely(err < 0)) {
879
				trace_xdp_exception(vi->dev, xdp_prog, act);
880 881 882 883
				if (unlikely(xdp_page != page))
					put_page(xdp_page);
				goto err_xdp;
			}
884
			*xdp_xmit |= VIRTIO_XDP_TX;
885
			if (unlikely(xdp_page != page))
886
				put_page(page);
J
John Fastabend 已提交
887 888
			rcu_read_unlock();
			goto xdp_xmit;
889
		case XDP_REDIRECT:
890
			stats->xdp_redirects++;
891 892 893 894 895 896
			err = xdp_do_redirect(dev, &xdp, xdp_prog);
			if (err) {
				if (unlikely(xdp_page != page))
					put_page(xdp_page);
				goto err_xdp;
			}
897
			*xdp_xmit |= VIRTIO_XDP_REDIR;
898
			if (unlikely(xdp_page != page))
899
				put_page(page);
900 901
			rcu_read_unlock();
			goto xdp_xmit;
J
John Fastabend 已提交
902
		default:
903
			bpf_warn_invalid_xdp_action(act);
904
			/* fall through */
905 906
		case XDP_ABORTED:
			trace_xdp_exception(vi->dev, xdp_prog, act);
907
			/* fall through */
908
		case XDP_DROP:
909 910
			if (unlikely(xdp_page != page))
				__free_pages(xdp_page, 0);
J
John Fastabend 已提交
911
			goto err_xdp;
J
John Fastabend 已提交
912
		}
J
John Fastabend 已提交
913 914
	}
	rcu_read_unlock();
915

916 917
	truesize = mergeable_ctx_to_truesize(ctx);
	if (unlikely(len > truesize)) {
918
		pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
919 920 921 922
			 dev->name, len, (unsigned long)ctx);
		dev->stats.rx_length_errors++;
		goto err_skb;
	}
923

924
	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog);
J
John Fastabend 已提交
925
	curr_skb = head_skb;
926

927 928
	if (unlikely(!curr_skb))
		goto err_skb;
929
	while (--num_buf) {
930 931
		int num_skb_frags;

932
		buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
933
		if (unlikely(!buf)) {
934
			pr_debug("%s: rx error: %d buffers out of %d missing\n",
M
Michael S. Tsirkin 已提交
935
				 dev->name, num_buf,
936 937
				 virtio16_to_cpu(vi->vdev,
						 hdr->num_buffers));
938 939
			dev->stats.rx_length_errors++;
			goto err_buf;
940
		}
941

942
		stats->bytes += len;
943
		page = virt_to_head_page(buf);
944 945 946

		truesize = mergeable_ctx_to_truesize(ctx);
		if (unlikely(len > truesize)) {
947
			pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
948 949 950 951
				 dev->name, len, (unsigned long)ctx);
			dev->stats.rx_length_errors++;
			goto err_skb;
		}
952 953

		num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
954 955
		if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
			struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
956 957 958

			if (unlikely(!nskb))
				goto err_skb;
959 960 961 962 963 964 965 966 967 968 969
			if (curr_skb == head_skb)
				skb_shinfo(curr_skb)->frag_list = nskb;
			else
				curr_skb->next = nskb;
			curr_skb = nskb;
			head_skb->truesize += nskb->truesize;
			num_skb_frags = 0;
		}
		if (curr_skb != head_skb) {
			head_skb->data_len += len;
			head_skb->len += len;
970
			head_skb->truesize += truesize;
971
		}
972
		offset = buf - page_address(page);
973 974 975
		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
			put_page(page);
			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
976
					     len, truesize);
977 978
		} else {
			skb_add_rx_frag(curr_skb, num_skb_frags, page,
979
					offset, len, truesize);
980
		}
981 982
	}

J
Johannes Berg 已提交
983
	ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
984 985
	return head_skb;

J
John Fastabend 已提交
986 987
err_xdp:
	rcu_read_unlock();
988
	stats->xdp_drops++;
989 990
err_skb:
	put_page(page);
991
	while (num_buf-- > 1) {
992 993
		buf = virtqueue_get_buf(rq->vq, &len);
		if (unlikely(!buf)) {
994 995 996 997 998
			pr_debug("%s: rx error: %d buffers missing\n",
				 dev->name, num_buf);
			dev->stats.rx_length_errors++;
			break;
		}
999
		stats->bytes += len;
1000
		page = virt_to_head_page(buf);
1001
		put_page(page);
1002
	}
1003
err_buf:
1004
	stats->drops++;
1005
	dev_kfree_skb(head_skb);
J
John Fastabend 已提交
1006
xdp_xmit:
1007
	return NULL;
1008 1009
}

1010 1011
static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
			void *buf, unsigned int len, void **ctx,
1012
			unsigned int *xdp_xmit,
1013
			struct virtnet_rq_stats *stats)
1014
{
1015
	struct net_device *dev = vi->dev;
1016
	struct sk_buff *skb;
1017
	struct virtio_net_hdr_mrg_rxbuf *hdr;
1018

1019
	if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
1020 1021
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
1022
		if (vi->mergeable_rx_bufs) {
1023
			put_page(virt_to_head_page(buf));
1024
		} else if (vi->big_packets) {
1025
			give_pages(rq, buf);
1026
		} else {
1027
			put_page(virt_to_head_page(buf));
1028
		}
1029
		return;
1030
	}
1031

1032
	if (vi->mergeable_rx_bufs)
1033
		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
1034
					stats);
1035
	else if (vi->big_packets)
1036
		skb = receive_big(dev, vi, rq, buf, len, stats);
1037
	else
1038
		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);
1039 1040

	if (unlikely(!skb))
1041
		return;
1042

1043
	hdr = skb_vnet_hdr(skb);
1044

1045
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
1046
		skb->ip_summed = CHECKSUM_UNNECESSARY;
R
Rusty Russell 已提交
1047

1048 1049 1050 1051 1052 1053
	if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
				  virtio_is_little_endian(vi->vdev))) {
		net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
				     dev->name, hdr->hdr.gso_type,
				     hdr->hdr.gso_size);
		goto frame_err;
R
Rusty Russell 已提交
1054 1055
	}

1056
	skb_record_rx_queue(skb, vq2rxq(rq->vq));
1057 1058 1059 1060
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

E
Eric Dumazet 已提交
1061
	napi_gro_receive(&rq->napi, skb);
1062
	return;
R
Rusty Russell 已提交
1063 1064 1065 1066 1067 1068

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

1069 1070 1071 1072 1073
/* Unlike mergeable buffers, all buffers are allocated to the
 * same size, except for the headroom. For this reason we do
 * not need to use  mergeable_len_to_ctx here - it is enough
 * to store the headroom as the context ignoring the truesize.
 */
M
Michael S. Tsirkin 已提交
1074 1075
static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
			     gfp_t gfp)
R
Rusty Russell 已提交
1076
{
1077 1078
	struct page_frag *alloc_frag = &rq->alloc_frag;
	char *buf;
1079
	unsigned int xdp_headroom = virtnet_get_headroom(vi);
1080
	void *ctx = (void *)(unsigned long)xdp_headroom;
1081
	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
1082
	int err;
1083

1084 1085 1086
	len = SKB_DATA_ALIGN(len) +
	      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
1087
		return -ENOMEM;
R
Rusty Russell 已提交
1088

1089 1090 1091 1092 1093
	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
	get_page(alloc_frag->page);
	alloc_frag->offset += len;
	sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
		    vi->hdr_len + GOOD_PACKET_LEN);
1094
	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1095
	if (err < 0)
1096
		put_page(virt_to_head_page(buf));
1097 1098
	return err;
}
1099

1100 1101
static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
			   gfp_t gfp)
1102 1103 1104 1105 1106
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

1107 1108
	sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);

1109
	/* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
1110
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
1111
		first = get_a_page(rq, gfp);
1112 1113
		if (!first) {
			if (list)
1114
				give_pages(rq, list);
1115
			return -ENOMEM;
1116
		}
1117
		sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
1118

1119 1120 1121 1122
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
R
Rusty Russell 已提交
1123

1124
	first = get_a_page(rq, gfp);
1125
	if (!first) {
1126
		give_pages(rq, list);
1127 1128 1129 1130
		return -ENOMEM;
	}
	p = page_address(first);

1131
	/* rq->sg[0], rq->sg[1] share the same page */
1132 1133
	/* a separated rq->sg[0] for header - required in case !any_header_sg */
	sg_set_buf(&rq->sg[0], p, vi->hdr_len);
1134

1135
	/* rq->sg[1] for data packet, from offset */
1136
	offset = sizeof(struct padded_vnet_hdr);
1137
	sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
1138 1139 1140

	/* chain first in list head */
	first->private = (unsigned long)list;
1141 1142
	err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
				  first, gfp);
1143
	if (err < 0)
1144
		give_pages(rq, first);
1145 1146

	return err;
R
Rusty Russell 已提交
1147 1148
}

1149
static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
1150 1151
					  struct ewma_pkt_len *avg_pkt_len,
					  unsigned int room)
1152
{
1153
	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1154 1155
	unsigned int len;

1156 1157 1158 1159
	if (room)
		return PAGE_SIZE - room;

	len = hdr_len +	clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
1160
				rq->min_buf_len, PAGE_SIZE - hdr_len);
1161

1162
	return ALIGN(len, L1_CACHE_BYTES);
1163 1164
}

1165 1166
static int add_recvbuf_mergeable(struct virtnet_info *vi,
				 struct receive_queue *rq, gfp_t gfp)
1167
{
1168
	struct page_frag *alloc_frag = &rq->alloc_frag;
1169
	unsigned int headroom = virtnet_get_headroom(vi);
1170 1171
	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1172
	char *buf;
1173
	void *ctx;
1174
	int err;
1175
	unsigned int len, hole;
1176

1177 1178 1179 1180 1181 1182
	/* Extra tailroom is needed to satisfy XDP's assumption. This
	 * means rx frags coalescing won't work, but consider we've
	 * disabled GSO for XDP, it won't be a big issue.
	 */
	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
	if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
1183
		return -ENOMEM;
1184

1185
	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1186
	buf += headroom; /* advance address leaving hole at front of pkt */
1187
	get_page(alloc_frag->page);
1188
	alloc_frag->offset += len + room;
1189
	hole = alloc_frag->size - alloc_frag->offset;
1190
	if (hole < len + room) {
1191 1192
		/* To avoid internal fragmentation, if there is very likely not
		 * enough space for another buffer, add the remaining space to
1193
		 * the current buffer.
1194
		 */
1195 1196 1197
		len += hole;
		alloc_frag->offset += hole;
	}
1198

1199
	sg_init_one(rq->sg, buf, len);
1200
	ctx = mergeable_len_to_ctx(len, headroom);
1201
	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1202
	if (err < 0)
1203
		put_page(virt_to_head_page(buf));
1204

1205 1206
	return err;
}
1207

1208 1209 1210 1211 1212 1213 1214
/*
 * Returns false if we couldn't fill entirely (OOM).
 *
 * Normally run in the receive path, but can also be run from ndo_open
 * before we're receiving packets, or from refill_work which is
 * careful to disable receiving (using napi_disable).
 */
M
Michael S. Tsirkin 已提交
1215 1216
static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
			  gfp_t gfp)
1217 1218
{
	int err;
1219
	bool oom;
1220

1221 1222
	do {
		if (vi->mergeable_rx_bufs)
1223
			err = add_recvbuf_mergeable(vi, rq, gfp);
1224
		else if (vi->big_packets)
1225
			err = add_recvbuf_big(vi, rq, gfp);
1226
		else
M
Michael S. Tsirkin 已提交
1227
			err = add_recvbuf_small(vi, rq, gfp);
1228

1229
		oom = err == -ENOMEM;
1230
		if (err)
1231
			break;
1232
	} while (rq->vq->num_free);
T
Toshiaki Makita 已提交
1233 1234
	if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
		u64_stats_update_begin(&rq->stats.syncp);
1235
		rq->stats.kicks++;
T
Toshiaki Makita 已提交
1236 1237 1238
		u64_stats_update_end(&rq->stats.syncp);
	}

1239
	return !oom;
1240 1241
}

1242
static void skb_recv_done(struct virtqueue *rvq)
R
Rusty Russell 已提交
1243 1244
{
	struct virtnet_info *vi = rvq->vdev->priv;
J
Jason Wang 已提交
1245
	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
1246

1247
	virtqueue_napi_schedule(&rq->napi, rvq);
R
Rusty Russell 已提交
1248 1249
}

1250
static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
1251
{
1252
	napi_enable(napi);
1253 1254

	/* If all buffers were filled by other side before we napi_enabled, we
1255 1256 1257 1258 1259 1260
	 * won't get another interrupt, so process any outstanding packets now.
	 * Call local_bh_enable after to trigger softIRQ processing.
	 */
	local_bh_disable();
	virtqueue_napi_schedule(napi, vq);
	local_bh_enable();
1261 1262
}

W
Willem de Bruijn 已提交
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
static void virtnet_napi_tx_enable(struct virtnet_info *vi,
				   struct virtqueue *vq,
				   struct napi_struct *napi)
{
	if (!napi->weight)
		return;

	/* Tx napi touches cachelines on the cpu handling tx interrupts. Only
	 * enable the feature if this is likely affine with the transmit path.
	 */
	if (!vi->affinity_hint_set) {
		napi->weight = 0;
		return;
	}

	return virtnet_napi_enable(vq, napi);
}

1281 1282 1283 1284 1285 1286
static void virtnet_napi_tx_disable(struct napi_struct *napi)
{
	if (napi->weight)
		napi_disable(napi);
}

1287 1288
static void refill_work(struct work_struct *work)
{
1289 1290
	struct virtnet_info *vi =
		container_of(work, struct virtnet_info, refill.work);
1291
	bool still_empty;
J
Jason Wang 已提交
1292 1293
	int i;

1294
	for (i = 0; i < vi->curr_queue_pairs; i++) {
J
Jason Wang 已提交
1295
		struct receive_queue *rq = &vi->rq[i];
1296

J
Jason Wang 已提交
1297
		napi_disable(&rq->napi);
M
Michael S. Tsirkin 已提交
1298
		still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
1299
		virtnet_napi_enable(rq->vq, &rq->napi);
1300

J
Jason Wang 已提交
1301 1302 1303 1304 1305 1306
		/* In theory, this can happen: if we don't get any buffers in
		 * we will *never* try to fill again.
		 */
		if (still_empty)
			schedule_delayed_work(&vi->refill, HZ/2);
	}
1307 1308
}

1309 1310
static int virtnet_receive(struct receive_queue *rq, int budget,
			   unsigned int *xdp_xmit)
R
Rusty Russell 已提交
1311
{
1312
	struct virtnet_info *vi = rq->vq->vdev->priv;
1313
	struct virtnet_rq_stats stats = {};
1314
	unsigned int len;
1315
	void *buf;
1316
	int i;
R
Rusty Russell 已提交
1317

1318
	if (!vi->big_packets || vi->mergeable_rx_bufs) {
1319 1320
		void *ctx;

1321
		while (stats.packets < budget &&
1322
		       (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
1323
			receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);
1324
			stats.packets++;
1325 1326
		}
	} else {
1327
		while (stats.packets < budget &&
1328
		       (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
1329
			receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);
1330
			stats.packets++;
1331
		}
R
Rusty Russell 已提交
1332 1333
	}

1334
	if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
M
Michael S. Tsirkin 已提交
1335
		if (!try_fill_recv(vi, rq, GFP_ATOMIC))
1336
			schedule_delayed_work(&vi->refill, 0);
1337
	}
R
Rusty Russell 已提交
1338

T
Toshiaki Makita 已提交
1339
	u64_stats_update_begin(&rq->stats.syncp);
1340 1341 1342 1343
	for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) {
		size_t offset = virtnet_rq_stats_desc[i].offset;
		u64 *item;

1344 1345
		item = (u64 *)((u8 *)&rq->stats + offset);
		*item += *(u64 *)((u8 *)&stats + offset);
1346
	}
T
Toshiaki Makita 已提交
1347
	u64_stats_update_end(&rq->stats.syncp);
J
Jason Wang 已提交
1348

1349
	return stats.packets;
1350 1351
}

1352
static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
1353 1354 1355 1356
{
	unsigned int len;
	unsigned int packets = 0;
	unsigned int bytes = 0;
1357
	void *ptr;
1358

1359 1360 1361
	while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
		if (likely(!is_xdp_frame(ptr))) {
			struct sk_buff *skb = ptr;
1362

1363
			pr_debug("Sent skb %p\n", skb);
1364

1365 1366 1367 1368
			bytes += skb->len;
			napi_consume_skb(skb, in_napi);
		} else {
			struct xdp_frame *frame = ptr_to_xdp(ptr);
1369

1370 1371 1372 1373
			bytes += frame->len;
			xdp_return_frame(frame);
		}
		packets++;
1374 1375 1376 1377 1378 1379 1380 1381
	}

	/* Avoid overhead when no packets have been processed
	 * happens when called speculatively from start_xmit.
	 */
	if (!packets)
		return;

T
Toshiaki Makita 已提交
1382 1383 1384 1385
	u64_stats_update_begin(&sq->stats.syncp);
	sq->stats.bytes += bytes;
	sq->stats.packets += packets;
	u64_stats_update_end(&sq->stats.syncp);
1386 1387
}

1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
{
	if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
		return false;
	else if (q < vi->curr_queue_pairs)
		return true;
	else
		return false;
}

1398 1399 1400 1401 1402 1403 1404
static void virtnet_poll_cleantx(struct receive_queue *rq)
{
	struct virtnet_info *vi = rq->vq->vdev->priv;
	unsigned int index = vq2rxq(rq->vq);
	struct send_queue *sq = &vi->sq[index];
	struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);

1405
	if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
1406 1407 1408
		return;

	if (__netif_tx_trylock(txq)) {
1409
		free_old_xmit_skbs(sq, true);
1410 1411 1412 1413 1414 1415 1416
		__netif_tx_unlock(txq);
	}

	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
		netif_tx_wake_queue(txq);
}

1417 1418 1419 1420
static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct receive_queue *rq =
		container_of(napi, struct receive_queue, napi);
1421 1422
	struct virtnet_info *vi = rq->vq->vdev->priv;
	struct send_queue *sq;
1423
	unsigned int received;
1424
	unsigned int xdp_xmit = 0;
1425

1426 1427
	virtnet_poll_cleantx(rq);

J
Jason Wang 已提交
1428
	received = virtnet_receive(rq, budget, &xdp_xmit);
1429

1430
	/* Out of packets? */
1431 1432
	if (received < budget)
		virtqueue_napi_complete(napi, rq->vq, received);
R
Rusty Russell 已提交
1433

1434 1435 1436 1437
	if (xdp_xmit & VIRTIO_XDP_REDIR)
		xdp_do_flush_map();

	if (xdp_xmit & VIRTIO_XDP_TX) {
1438
		sq = virtnet_xdp_sq(vi);
T
Toshiaki Makita 已提交
1439 1440 1441 1442 1443
		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
			u64_stats_update_begin(&sq->stats.syncp);
			sq->stats.kicks++;
			u64_stats_update_end(&sq->stats.syncp);
		}
1444
	}
J
Jason Wang 已提交
1445

R
Rusty Russell 已提交
1446 1447 1448
	return received;
}

J
Jason Wang 已提交
1449 1450 1451
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
1452
	int i, err;
J
Jason Wang 已提交
1453

1454 1455 1456
	for (i = 0; i < vi->max_queue_pairs; i++) {
		if (i < vi->curr_queue_pairs)
			/* Make sure we have some buffers: if oom use wq. */
M
Michael S. Tsirkin 已提交
1457
			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1458
				schedule_delayed_work(&vi->refill, 0);
1459 1460 1461 1462 1463

		err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i);
		if (err < 0)
			return err;

1464 1465 1466 1467 1468 1469 1470
		err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,
						 MEM_TYPE_PAGE_SHARED, NULL);
		if (err < 0) {
			xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
			return err;
		}

1471
		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
W
Willem de Bruijn 已提交
1472
		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
J
Jason Wang 已提交
1473 1474 1475 1476 1477
	}

	return 0;
}

W
Willem de Bruijn 已提交
1478 1479 1480 1481
static int virtnet_poll_tx(struct napi_struct *napi, int budget)
{
	struct send_queue *sq = container_of(napi, struct send_queue, napi);
	struct virtnet_info *vi = sq->vq->vdev->priv;
1482 1483
	unsigned int index = vq2txq(sq->vq);
	struct netdev_queue *txq;
W
Willem de Bruijn 已提交
1484

1485 1486 1487 1488 1489 1490 1491
	if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
		/* We don't need to enable cb for XDP */
		napi_complete_done(napi, 0);
		return 0;
	}

	txq = netdev_get_tx_queue(vi->dev, index);
W
Willem de Bruijn 已提交
1492
	__netif_tx_lock(txq, raw_smp_processor_id());
1493
	free_old_xmit_skbs(sq, true);
W
Willem de Bruijn 已提交
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
	__netif_tx_unlock(txq);

	virtqueue_napi_complete(napi, sq->vq, 0);

	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
		netif_tx_wake_queue(txq);

	return 0;
}

1504
static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
R
Rusty Russell 已提交
1505
{
1506
	struct virtio_net_hdr_mrg_rxbuf *hdr;
R
Rusty Russell 已提交
1507
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
1508
	struct virtnet_info *vi = sq->vq->vdev->priv;
1509
	int num_sg;
1510
	unsigned hdr_len = vi->hdr_len;
1511
	bool can_push;
R
Rusty Russell 已提交
1512

J
Johannes Berg 已提交
1513
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
1514 1515 1516 1517 1518 1519 1520

	can_push = vi->any_header_sg &&
		!((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
		!skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
	/* Even if we can, don't push here yet as this would skew
	 * csum_start offset below. */
	if (can_push)
1521
		hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
1522 1523
	else
		hdr = skb_vnet_hdr(skb);
R
Rusty Russell 已提交
1524

1525
	if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
1526 1527
				    virtio_is_little_endian(vi->vdev), false,
				    0))
1528
		BUG();
R
Rusty Russell 已提交
1529

1530
	if (vi->mergeable_rx_bufs)
1531
		hdr->num_buffers = 0;
1532

1533
	sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
1534 1535 1536
	if (can_push) {
		__skb_push(skb, hdr_len);
		num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
1537 1538
		if (unlikely(num_sg < 0))
			return num_sg;
1539 1540 1541 1542
		/* Pull header back to avoid skew in tx bytes calculations. */
		__skb_pull(skb, hdr_len);
	} else {
		sg_set_buf(sq->sg, hdr, hdr_len);
1543 1544 1545 1546
		num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
		if (unlikely(num_sg < 0))
			return num_sg;
		num_sg++;
1547
	}
1548
	return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
1549 1550
}

1551
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
1552 1553
{
	struct virtnet_info *vi = netdev_priv(dev);
J
Jason Wang 已提交
1554 1555
	int qnum = skb_get_queue_mapping(skb);
	struct send_queue *sq = &vi->sq[qnum];
1556
	int err;
1557
	struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1558
	bool kick = !netdev_xmit_more();
W
Willem de Bruijn 已提交
1559
	bool use_napi = sq->napi.weight;
1560 1561

	/* Free up any pending old buffers before queueing new ones. */
1562
	free_old_xmit_skbs(sq, false);
1563

1564 1565 1566
	if (use_napi && kick)
		virtqueue_enable_cb_delayed(sq->vq);

1567 1568 1569
	/* timestamp packet in software */
	skb_tx_timestamp(skb);

1570
	/* Try to transmit */
1571
	err = xmit_skb(sq, skb);
1572

1573
	/* This should not happen! */
1574
	if (unlikely(err)) {
1575 1576 1577
		dev->stats.tx_fifo_errors++;
		if (net_ratelimit())
			dev_warn(&dev->dev,
1578 1579
				 "Unexpected TXQ (%d) queue failure: %d\n",
				 qnum, err);
1580
		dev->stats.tx_dropped++;
1581
		dev_kfree_skb_any(skb);
1582
		return NETDEV_TX_OK;
R
Rusty Russell 已提交
1583
	}
1584

1585
	/* Don't wait up for transmitted skbs to be freed. */
W
Willem de Bruijn 已提交
1586 1587 1588 1589
	if (!use_napi) {
		skb_orphan(skb);
		nf_reset(skb);
	}
1590

1591 1592 1593 1594 1595 1596 1597 1598 1599
	/* If running out of space, stop queue to avoid getting packets that we
	 * are then unable to transmit.
	 * An alternative would be to force queuing layer to requeue the skb by
	 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
	 * returned in a normal path of operation: it means that driver is not
	 * maintaining the TX queue stop/start state properly, and causes
	 * the stack to do a non-trivial amount of useless work.
	 * Since most packets only take 1 or 2 ring slots, stopping the queue
	 * early means 16 slots are typically wasted.
1600
	 */
1601
	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
J
Jason Wang 已提交
1602
		netif_stop_subqueue(dev, qnum);
W
Willem de Bruijn 已提交
1603 1604
		if (!use_napi &&
		    unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1605
			/* More just got used, free them then recheck. */
1606
			free_old_xmit_skbs(sq, false);
1607
			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
J
Jason Wang 已提交
1608
				netif_start_subqueue(dev, qnum);
1609
				virtqueue_disable_cb(sq->vq);
1610 1611
			}
		}
1612
	}
1613

T
Toshiaki Makita 已提交
1614 1615 1616 1617 1618 1619 1620
	if (kick || netif_xmit_stopped(txq)) {
		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
			u64_stats_update_begin(&sq->stats.syncp);
			sq->stats.kicks++;
			u64_stats_update_end(&sq->stats.syncp);
		}
	}
R
Rusty Russell 已提交
1621

1622
	return NETDEV_TX_OK;
1623 1624
}

1625 1626 1627
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
S
stephen hemminger 已提交
1628
 * never fail unless improperly formatted.
1629 1630
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
1631
				 struct scatterlist *out)
1632
{
1633
	struct scatterlist *sgs[4], hdr, stat;
1634
	unsigned out_num = 0, tmp;
1635 1636

	/* Caller should know better */
1637
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
1638

1639 1640 1641
	vi->ctrl->status = ~0;
	vi->ctrl->hdr.class = class;
	vi->ctrl->hdr.cmd = cmd;
1642
	/* Add header */
1643
	sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
1644
	sgs[out_num++] = &hdr;
1645

1646 1647
	if (out)
		sgs[out_num++] = out;
1648

1649
	/* Add return status. */
1650
	sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
1651
	sgs[out_num] = &stat;
1652

1653
	BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
1654
	virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
1655

1656
	if (unlikely(!virtqueue_kick(vi->cvq)))
1657
		return vi->ctrl->status == VIRTIO_NET_OK;
1658 1659 1660 1661

	/* Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
1662 1663
	while (!virtqueue_get_buf(vi->cvq, &tmp) &&
	       !virtqueue_is_broken(vi->cvq))
1664 1665
		cpu_relax();

1666
	return vi->ctrl->status == VIRTIO_NET_OK;
1667 1668
}

1669 1670 1671 1672
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
1673
	int ret;
1674
	struct sockaddr *addr;
1675
	struct scatterlist sg;
1676

1677 1678 1679
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
		return -EOPNOTSUPP;

1680
	addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
1681 1682 1683 1684
	if (!addr)
		return -ENOMEM;

	ret = eth_prepare_mac_addr_change(dev, addr);
1685
	if (ret)
1686
		goto out;
1687

1688 1689 1690
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
		sg_init_one(&sg, addr->sa_data, dev->addr_len);
		if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1691
					  VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1692 1693
			dev_warn(&vdev->dev,
				 "Failed to set mac address by vq command.\n");
1694 1695
			ret = -EINVAL;
			goto out;
1696
		}
1697 1698
	} else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
		   !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1699 1700 1701 1702 1703 1704 1705
		unsigned int i;

		/* Naturally, this has an atomicity problem. */
		for (i = 0; i < dev->addr_len; i++)
			virtio_cwrite8(vdev,
				       offsetof(struct virtio_net_config, mac) +
				       i, addr->sa_data[i]);
1706 1707 1708
	}

	eth_commit_mac_addr_change(dev, p);
1709
	ret = 0;
1710

1711 1712 1713
out:
	kfree(addr);
	return ret;
1714 1715
}

1716 1717
static void virtnet_stats(struct net_device *dev,
			  struct rtnl_link_stats64 *tot)
1718 1719 1720
{
	struct virtnet_info *vi = netdev_priv(dev);
	unsigned int start;
T
Toshiaki Makita 已提交
1721
	int i;
1722

T
Toshiaki Makita 已提交
1723
	for (i = 0; i < vi->max_queue_pairs; i++) {
1724
		u64 tpackets, tbytes, rpackets, rbytes, rdrops;
T
Toshiaki Makita 已提交
1725 1726
		struct receive_queue *rq = &vi->rq[i];
		struct send_queue *sq = &vi->sq[i];
1727 1728

		do {
T
Toshiaki Makita 已提交
1729 1730 1731 1732
			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
			tpackets = sq->stats.packets;
			tbytes   = sq->stats.bytes;
		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
1733 1734

		do {
T
Toshiaki Makita 已提交
1735
			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
1736 1737 1738
			rpackets = rq->stats.packets;
			rbytes   = rq->stats.bytes;
			rdrops   = rq->stats.drops;
T
Toshiaki Makita 已提交
1739
		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
1740 1741 1742 1743 1744

		tot->rx_packets += rpackets;
		tot->tx_packets += tpackets;
		tot->rx_bytes   += rbytes;
		tot->tx_bytes   += tbytes;
1745
		tot->rx_dropped += rdrops;
1746 1747 1748
	}

	tot->tx_dropped = dev->stats.tx_dropped;
1749
	tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1750 1751 1752 1753
	tot->rx_length_errors = dev->stats.rx_length_errors;
	tot->rx_frame_errors = dev->stats.rx_frame_errors;
}

1754 1755 1756 1757
static void virtnet_ack_link_announce(struct virtnet_info *vi)
{
	rtnl_lock();
	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1758
				  VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1759 1760 1761 1762
		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
	rtnl_unlock();
}

1763
static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
J
Jason Wang 已提交
1764 1765 1766 1767 1768 1769 1770
{
	struct scatterlist sg;
	struct net_device *dev = vi->dev;

	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
		return 0;

1771 1772
	vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
	sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));
J
Jason Wang 已提交
1773 1774

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1775
				  VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
J
Jason Wang 已提交
1776 1777 1778
		dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
			 queue_pairs);
		return -EINVAL;
1779
	} else {
J
Jason Wang 已提交
1780
		vi->curr_queue_pairs = queue_pairs;
1781 1782 1783
		/* virtnet_open() will refill when device is going to up. */
		if (dev->flags & IFF_UP)
			schedule_delayed_work(&vi->refill, 0);
1784
	}
J
Jason Wang 已提交
1785 1786 1787 1788

	return 0;
}

1789 1790 1791 1792 1793 1794 1795 1796 1797 1798
static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
{
	int err;

	rtnl_lock();
	err = _virtnet_set_queues(vi, queue_pairs);
	rtnl_unlock();
	return err;
}

R
Rusty Russell 已提交
1799 1800 1801
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
J
Jason Wang 已提交
1802
	int i;
R
Rusty Russell 已提交
1803

1804 1805
	/* Make sure refill_work doesn't re-enable napi! */
	cancel_delayed_work_sync(&vi->refill);
J
Jason Wang 已提交
1806

W
Willem de Bruijn 已提交
1807
	for (i = 0; i < vi->max_queue_pairs; i++) {
1808
		xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
J
Jason Wang 已提交
1809
		napi_disable(&vi->rq[i].napi);
1810
		virtnet_napi_tx_disable(&vi->sq[i].napi);
W
Willem de Bruijn 已提交
1811
	}
R
Rusty Russell 已提交
1812 1813 1814 1815

	return 0;
}

1816 1817 1818
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
1819 1820
	struct scatterlist sg[2];
	struct virtio_net_ctrl_mac *mac_data;
J
Jiri Pirko 已提交
1821
	struct netdev_hw_addr *ha;
1822
	int uc_count;
1823
	int mc_count;
1824 1825
	void *buf;
	int i;
1826

S
stephen hemminger 已提交
1827
	/* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1828 1829 1830
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

1831 1832
	vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
	vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
1833

1834
	sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));
1835 1836

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1837
				  VIRTIO_NET_CTRL_RX_PROMISC, sg))
1838
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
1839
			 vi->ctrl->promisc ? "en" : "dis");
1840

1841
	sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));
1842 1843

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1844
				  VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
1845
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
1846
			 vi->ctrl->allmulti ? "en" : "dis");
1847

1848
	uc_count = netdev_uc_count(dev);
1849
	mc_count = netdev_mc_count(dev);
1850
	/* MAC filter - use one buffer for both lists */
1851 1852 1853
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
1854
	if (!buf)
1855 1856
		return;

1857 1858
	sg_init_table(sg, 2);

1859
	/* Store the unicast list and count in the front of the buffer */
M
Michael S. Tsirkin 已提交
1860
	mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
J
Jiri Pirko 已提交
1861
	i = 0;
1862
	netdev_for_each_uc_addr(ha, dev)
J
Jiri Pirko 已提交
1863
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1864 1865

	sg_set_buf(&sg[0], mac_data,
1866
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
1867 1868

	/* multicast list and count fill the end */
1869
	mac_data = (void *)&mac_data->macs[uc_count][0];
1870

M
Michael S. Tsirkin 已提交
1871
	mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
1872
	i = 0;
1873 1874
	netdev_for_each_mc_addr(ha, dev)
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1875 1876

	sg_set_buf(&sg[1], mac_data,
1877
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
1878 1879

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1880
				  VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
1881
		dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
1882 1883

	kfree(buf);
1884 1885
}

1886 1887
static int virtnet_vlan_rx_add_vid(struct net_device *dev,
				   __be16 proto, u16 vid)
1888 1889 1890 1891
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

1892
	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
1893
	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
1894 1895

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1896
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg))
1897
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
1898
	return 0;
1899 1900
}

1901 1902
static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
				    __be16 proto, u16 vid)
1903 1904 1905 1906
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

1907
	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
1908
	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
1909 1910

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1911
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg))
1912
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
1913
	return 0;
1914 1915
}

1916
static void virtnet_clean_affinity(struct virtnet_info *vi)
J
Jason Wang 已提交
1917 1918 1919
{
	int i;

1920 1921
	if (vi->affinity_hint_set) {
		for (i = 0; i < vi->max_queue_pairs; i++) {
1922 1923
			virtqueue_set_affinity(vi->rq[i].vq, NULL);
			virtqueue_set_affinity(vi->sq[i].vq, NULL);
1924 1925
		}

1926 1927 1928
		vi->affinity_hint_set = false;
	}
}
1929

1930 1931
static void virtnet_set_affinity(struct virtnet_info *vi)
{
1932 1933 1934 1935 1936 1937 1938 1939
	cpumask_var_t mask;
	int stragglers;
	int group_size;
	int i, j, cpu;
	int num_cpu;
	int stride;

	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
1940
		virtnet_clean_affinity(vi);
1941
		return;
J
Jason Wang 已提交
1942 1943
	}

1944 1945 1946 1947 1948 1949
	num_cpu = num_online_cpus();
	stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);
	stragglers = num_cpu >= vi->curr_queue_pairs ?
			num_cpu % vi->curr_queue_pairs :
			0;
	cpu = cpumask_next(-1, cpu_online_mask);
1950

1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962
	for (i = 0; i < vi->curr_queue_pairs; i++) {
		group_size = stride + (i < stragglers ? 1 : 0);

		for (j = 0; j < group_size; j++) {
			cpumask_set_cpu(cpu, mask);
			cpu = cpumask_next_wrap(cpu, cpu_online_mask,
						nr_cpu_ids, false);
		}
		virtqueue_set_affinity(vi->rq[i].vq, mask);
		virtqueue_set_affinity(vi->sq[i].vq, mask);
		__netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, false);
		cpumask_clear(mask);
J
Jason Wang 已提交
1963 1964
	}

1965
	vi->affinity_hint_set = true;
1966
	free_cpumask_var(mask);
J
Jason Wang 已提交
1967 1968
}

1969
static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
1970
{
1971 1972 1973 1974 1975
	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
						   node);
	virtnet_set_affinity(vi);
	return 0;
}
1976

1977 1978 1979 1980 1981 1982 1983
static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
{
	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
						   node_dead);
	virtnet_set_affinity(vi);
	return 0;
}
1984

1985 1986 1987 1988 1989
static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
{
	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
						   node);

1990
	virtnet_clean_affinity(vi);
1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015
	return 0;
}

static enum cpuhp_state virtionet_online;

static int virtnet_cpu_notif_add(struct virtnet_info *vi)
{
	int ret;

	ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
	if (ret)
		return ret;
	ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
					       &vi->node_dead);
	if (!ret)
		return ret;
	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
	return ret;
}

static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
{
	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
	cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
					    &vi->node_dead);
J
Jason Wang 已提交
2016 2017
}

R
Rick Jones 已提交
2018 2019 2020 2021 2022
static void virtnet_get_ringparam(struct net_device *dev,
				struct ethtool_ringparam *ring)
{
	struct virtnet_info *vi = netdev_priv(dev);

J
Jason Wang 已提交
2023 2024
	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
R
Rick Jones 已提交
2025 2026 2027 2028
	ring->rx_pending = ring->rx_max_pending;
	ring->tx_pending = ring->tx_max_pending;
}

2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041

static void virtnet_get_drvinfo(struct net_device *dev,
				struct ethtool_drvinfo *info)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;

	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
	strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
	strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));

}

2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055
/* TODO: Eliminate OOO packets during switching */
static int virtnet_set_channels(struct net_device *dev,
				struct ethtool_channels *channels)
{
	struct virtnet_info *vi = netdev_priv(dev);
	u16 queue_pairs = channels->combined_count;
	int err;

	/* We don't support separate rx/tx channels.
	 * We don't allow setting 'other' channels.
	 */
	if (channels->rx_count || channels->tx_count || channels->other_count)
		return -EINVAL;

2056
	if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
2057 2058
		return -EINVAL;

J
John Fastabend 已提交
2059 2060 2061 2062 2063 2064 2065
	/* For now we don't support modifying channels while XDP is loaded
	 * also when XDP is loaded all RX queues have XDP programs so we only
	 * need to check a single RX queue.
	 */
	if (vi->rq[0].xdp_prog)
		return -EINVAL;

2066
	get_online_cpus();
2067
	err = _virtnet_set_queues(vi, queue_pairs);
2068 2069 2070 2071
	if (!err) {
		netif_set_real_num_tx_queues(dev, queue_pairs);
		netif_set_real_num_rx_queues(dev, queue_pairs);

2072
		virtnet_set_affinity(vi);
2073
	}
2074
	put_online_cpus();
2075 2076 2077 2078

	return err;
}

T
Toshiaki Makita 已提交
2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129
static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
{
	struct virtnet_info *vi = netdev_priv(dev);
	char *p = (char *)data;
	unsigned int i, j;

	switch (stringset) {
	case ETH_SS_STATS:
		for (i = 0; i < vi->curr_queue_pairs; i++) {
			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
				snprintf(p, ETH_GSTRING_LEN, "rx_queue_%u_%s",
					 i, virtnet_rq_stats_desc[j].desc);
				p += ETH_GSTRING_LEN;
			}
		}

		for (i = 0; i < vi->curr_queue_pairs; i++) {
			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
				snprintf(p, ETH_GSTRING_LEN, "tx_queue_%u_%s",
					 i, virtnet_sq_stats_desc[j].desc);
				p += ETH_GSTRING_LEN;
			}
		}
		break;
	}
}

static int virtnet_get_sset_count(struct net_device *dev, int sset)
{
	struct virtnet_info *vi = netdev_priv(dev);

	switch (sset) {
	case ETH_SS_STATS:
		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
					       VIRTNET_SQ_STATS_LEN);
	default:
		return -EOPNOTSUPP;
	}
}

static void virtnet_get_ethtool_stats(struct net_device *dev,
				      struct ethtool_stats *stats, u64 *data)
{
	struct virtnet_info *vi = netdev_priv(dev);
	unsigned int idx = 0, start, i, j;
	const u8 *stats_base;
	size_t offset;

	for (i = 0; i < vi->curr_queue_pairs; i++) {
		struct receive_queue *rq = &vi->rq[i];

2130
		stats_base = (u8 *)&rq->stats;
T
Toshiaki Makita 已提交
2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155
		do {
			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
				offset = virtnet_rq_stats_desc[j].offset;
				data[idx + j] = *(u64 *)(stats_base + offset);
			}
		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
		idx += VIRTNET_RQ_STATS_LEN;
	}

	for (i = 0; i < vi->curr_queue_pairs; i++) {
		struct send_queue *sq = &vi->sq[i];

		stats_base = (u8 *)&sq->stats;
		do {
			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
				offset = virtnet_sq_stats_desc[j].offset;
				data[idx + j] = *(u64 *)(stats_base + offset);
			}
		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
		idx += VIRTNET_SQ_STATS_LEN;
	}
}

2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168
static void virtnet_get_channels(struct net_device *dev,
				 struct ethtool_channels *channels)
{
	struct virtnet_info *vi = netdev_priv(dev);

	channels->combined_count = vi->curr_queue_pairs;
	channels->max_combined = vi->max_queue_pairs;
	channels->max_other = 0;
	channels->rx_count = 0;
	channels->tx_count = 0;
	channels->other_count = 0;
}

2169
/* Check if the user is trying to change anything besides speed/duplex */
2170 2171
static bool
virtnet_validate_ethtool_cmd(const struct ethtool_link_ksettings *cmd)
2172
{
2173 2174
	struct ethtool_link_ksettings diff1 = *cmd;
	struct ethtool_link_ksettings diff2 = {};
2175

2176 2177 2178
	/* cmd is always set so we need to clear it, validate the port type
	 * and also without autonegotiation we can ignore advertising
	 */
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
	diff1.base.speed = 0;
	diff2.base.port = PORT_OTHER;
	ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
	diff1.base.duplex = 0;
	diff1.base.cmd = 0;
	diff1.base.link_mode_masks_nwords = 0;

	return !memcmp(&diff1.base, &diff2.base, sizeof(diff1.base)) &&
		bitmap_empty(diff1.link_modes.supported,
			     __ETHTOOL_LINK_MODE_MASK_NBITS) &&
		bitmap_empty(diff1.link_modes.advertising,
			     __ETHTOOL_LINK_MODE_MASK_NBITS) &&
		bitmap_empty(diff1.link_modes.lp_advertising,
			     __ETHTOOL_LINK_MODE_MASK_NBITS);
2193 2194
}

2195 2196
static int virtnet_set_link_ksettings(struct net_device *dev,
				      const struct ethtool_link_ksettings *cmd)
2197 2198 2199 2200
{
	struct virtnet_info *vi = netdev_priv(dev);
	u32 speed;

2201
	speed = cmd->base.speed;
2202 2203
	/* don't allow custom speed and duplex */
	if (!ethtool_validate_speed(speed) ||
2204
	    !ethtool_validate_duplex(cmd->base.duplex) ||
2205 2206 2207
	    !virtnet_validate_ethtool_cmd(cmd))
		return -EINVAL;
	vi->speed = speed;
2208
	vi->duplex = cmd->base.duplex;
2209 2210 2211 2212

	return 0;
}

2213 2214
static int virtnet_get_link_ksettings(struct net_device *dev,
				      struct ethtool_link_ksettings *cmd)
2215 2216 2217
{
	struct virtnet_info *vi = netdev_priv(dev);

2218 2219 2220
	cmd->base.speed = vi->speed;
	cmd->base.duplex = vi->duplex;
	cmd->base.port = PORT_OTHER;
2221 2222 2223 2224

	return 0;
}

2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271
static int virtnet_set_coalesce(struct net_device *dev,
				struct ethtool_coalesce *ec)
{
	struct ethtool_coalesce ec_default = {
		.cmd = ETHTOOL_SCOALESCE,
		.rx_max_coalesced_frames = 1,
	};
	struct virtnet_info *vi = netdev_priv(dev);
	int i, napi_weight;

	if (ec->tx_max_coalesced_frames > 1)
		return -EINVAL;

	ec_default.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
	napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;

	/* disallow changes to fields not explicitly tested above */
	if (memcmp(ec, &ec_default, sizeof(ec_default)))
		return -EINVAL;

	if (napi_weight ^ vi->sq[0].napi.weight) {
		if (dev->flags & IFF_UP)
			return -EBUSY;
		for (i = 0; i < vi->max_queue_pairs; i++)
			vi->sq[i].napi.weight = napi_weight;
	}

	return 0;
}

static int virtnet_get_coalesce(struct net_device *dev,
				struct ethtool_coalesce *ec)
{
	struct ethtool_coalesce ec_default = {
		.cmd = ETHTOOL_GCOALESCE,
		.rx_max_coalesced_frames = 1,
	};
	struct virtnet_info *vi = netdev_priv(dev);

	memcpy(ec, &ec_default, sizeof(ec_default));

	if (vi->sq[0].napi.weight)
		ec->tx_max_coalesced_frames = 1;

	return 0;
}

2272 2273 2274 2275 2276 2277 2278 2279
static void virtnet_init_settings(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	vi->speed = SPEED_UNKNOWN;
	vi->duplex = DUPLEX_UNKNOWN;
}

2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297
static void virtnet_update_settings(struct virtnet_info *vi)
{
	u32 speed;
	u8 duplex;

	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
		return;

	speed = virtio_cread32(vi->vdev, offsetof(struct virtio_net_config,
						  speed));
	if (ethtool_validate_speed(speed))
		vi->speed = speed;
	duplex = virtio_cread8(vi->vdev, offsetof(struct virtio_net_config,
						  duplex));
	if (ethtool_validate_duplex(duplex))
		vi->duplex = duplex;
}

2298
static const struct ethtool_ops virtnet_ethtool_ops = {
2299
	.get_drvinfo = virtnet_get_drvinfo,
2300
	.get_link = ethtool_op_get_link,
R
Rick Jones 已提交
2301
	.get_ringparam = virtnet_get_ringparam,
T
Toshiaki Makita 已提交
2302 2303 2304
	.get_strings = virtnet_get_strings,
	.get_sset_count = virtnet_get_sset_count,
	.get_ethtool_stats = virtnet_get_ethtool_stats,
2305 2306
	.set_channels = virtnet_set_channels,
	.get_channels = virtnet_get_channels,
2307
	.get_ts_info = ethtool_op_get_ts_info,
2308 2309
	.get_link_ksettings = virtnet_get_link_ksettings,
	.set_link_ksettings = virtnet_set_link_ksettings,
2310 2311
	.set_coalesce = virtnet_set_coalesce,
	.get_coalesce = virtnet_get_coalesce,
2312 2313
};

2314 2315 2316 2317 2318 2319 2320 2321
static void virtnet_freeze_down(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;
	int i;

	/* Make sure no work handler is accessing the device */
	flush_work(&vi->config_work);

2322
	netif_tx_lock_bh(vi->dev);
2323
	netif_device_detach(vi->dev);
2324
	netif_tx_unlock_bh(vi->dev);
2325 2326 2327
	cancel_delayed_work_sync(&vi->refill);

	if (netif_running(vi->dev)) {
W
Willem de Bruijn 已提交
2328
		for (i = 0; i < vi->max_queue_pairs; i++) {
2329
			napi_disable(&vi->rq[i].napi);
2330
			virtnet_napi_tx_disable(&vi->sq[i].napi);
W
Willem de Bruijn 已提交
2331
		}
2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352
	}
}

static int init_vqs(struct virtnet_info *vi);

static int virtnet_restore_up(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;
	int err, i;

	err = init_vqs(vi);
	if (err)
		return err;

	virtio_device_ready(vdev);

	if (netif_running(vi->dev)) {
		for (i = 0; i < vi->curr_queue_pairs; i++)
			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
				schedule_delayed_work(&vi->refill, 0);

W
Willem de Bruijn 已提交
2353
		for (i = 0; i < vi->max_queue_pairs; i++) {
2354
			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
W
Willem de Bruijn 已提交
2355 2356 2357
			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
					       &vi->sq[i].napi);
		}
2358 2359
	}

2360
	netif_tx_lock_bh(vi->dev);
2361
	netif_device_attach(vi->dev);
2362
	netif_tx_unlock_bh(vi->dev);
2363 2364 2365
	return err;
}

2366 2367 2368
static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
{
	struct scatterlist sg;
2369
	vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);
2370

2371
	sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));
2372 2373 2374

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
				  VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
2375
		dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");
2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
		return -EINVAL;
	}

	return 0;
}

static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
{
	u64 offloads = 0;

	if (!vi->guest_offloads)
		return 0;

	return virtnet_set_guest_offloads(vi, offloads);
}

static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
{
	u64 offloads = vi->guest_offloads;

	if (!vi->guest_offloads)
		return 0;

	return virtnet_set_guest_offloads(vi, offloads);
}

2402 2403
static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
			   struct netlink_ext_ack *extack)
J
John Fastabend 已提交
2404 2405 2406 2407
{
	unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
	struct virtnet_info *vi = netdev_priv(dev);
	struct bpf_prog *old_prog;
2408
	u16 xdp_qp = 0, curr_qp;
2409
	int i, err;
J
John Fastabend 已提交
2410

2411 2412 2413 2414
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
	    && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
2415 2416 2417
		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) {
		NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing LRO/CSUM, disable LRO/CSUM first");
J
John Fastabend 已提交
2418 2419 2420 2421
		return -EOPNOTSUPP;
	}

	if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
2422
		NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
J
John Fastabend 已提交
2423 2424 2425 2426
		return -EINVAL;
	}

	if (dev->mtu > max_sz) {
2427
		NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
J
John Fastabend 已提交
2428 2429 2430 2431
		netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
		return -EINVAL;
	}

2432 2433 2434 2435 2436 2437
	curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
	if (prog)
		xdp_qp = nr_cpu_ids;

	/* XDP requires extra queues for XDP_TX */
	if (curr_qp + xdp_qp > vi->max_queue_pairs) {
2438
		NL_SET_ERR_MSG_MOD(extack, "Too few free TX rings available");
2439 2440 2441 2442 2443
		netdev_warn(dev, "request %i queues but max is %i\n",
			    curr_qp + xdp_qp, vi->max_queue_pairs);
		return -ENOMEM;
	}

2444 2445 2446 2447
	old_prog = rtnl_dereference(vi->rq[0].xdp_prog);
	if (!prog && !old_prog)
		return 0;

2448 2449 2450 2451 2452 2453
	if (prog) {
		prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);
		if (IS_ERR(prog))
			return PTR_ERR(prog);
	}

2454
	/* Make sure NAPI is not using any XDP TX queues for RX. */
2455 2456
	if (netif_running(dev)) {
		for (i = 0; i < vi->max_queue_pairs; i++) {
2457
			napi_disable(&vi->rq[i].napi);
2458 2459 2460
			virtnet_napi_tx_disable(&vi->sq[i].napi);
		}
	}
J
John Fastabend 已提交
2461

2462 2463 2464 2465 2466 2467 2468 2469
	if (!prog) {
		for (i = 0; i < vi->max_queue_pairs; i++) {
			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
			if (i == 0)
				virtnet_restore_guest_offloads(vi);
		}
		synchronize_net();
	}
J
John Fastabend 已提交
2470

2471 2472 2473
	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
	if (err)
		goto err;
2474
	netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
2475
	vi->xdp_queue_pairs = xdp_qp;
2476

2477 2478 2479 2480
	if (prog) {
		for (i = 0; i < vi->max_queue_pairs; i++) {
			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
			if (i == 0 && !old_prog)
2481 2482
				virtnet_clear_guest_offloads(vi);
		}
2483 2484 2485
	}

	for (i = 0; i < vi->max_queue_pairs; i++) {
J
John Fastabend 已提交
2486 2487
		if (old_prog)
			bpf_prog_put(old_prog);
2488
		if (netif_running(dev)) {
2489
			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2490 2491 2492
			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
					       &vi->sq[i].napi);
		}
J
John Fastabend 已提交
2493 2494 2495
	}

	return 0;
2496

2497
err:
2498 2499 2500 2501 2502 2503
	if (!prog) {
		virtnet_clear_guest_offloads(vi);
		for (i = 0; i < vi->max_queue_pairs; i++)
			rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);
	}

2504
	if (netif_running(dev)) {
2505
		for (i = 0; i < vi->max_queue_pairs; i++) {
2506
			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2507 2508 2509
			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
					       &vi->sq[i].napi);
		}
2510
	}
2511 2512 2513
	if (prog)
		bpf_prog_sub(prog, vi->max_queue_pairs - 1);
	return err;
J
John Fastabend 已提交
2514 2515
}

2516
static u32 virtnet_xdp_query(struct net_device *dev)
J
John Fastabend 已提交
2517 2518
{
	struct virtnet_info *vi = netdev_priv(dev);
2519
	const struct bpf_prog *xdp_prog;
J
John Fastabend 已提交
2520 2521 2522
	int i;

	for (i = 0; i < vi->max_queue_pairs; i++) {
2523 2524 2525
		xdp_prog = rtnl_dereference(vi->rq[i].xdp_prog);
		if (xdp_prog)
			return xdp_prog->aux->id;
J
John Fastabend 已提交
2526
	}
2527
	return 0;
J
John Fastabend 已提交
2528 2529
}

2530
static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
J
John Fastabend 已提交
2531 2532 2533
{
	switch (xdp->command) {
	case XDP_SETUP_PROG:
2534
		return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
J
John Fastabend 已提交
2535
	case XDP_QUERY_PROG:
2536
		xdp->prog_id = virtnet_xdp_query(dev);
J
John Fastabend 已提交
2537 2538 2539 2540 2541 2542
		return 0;
	default:
		return -EINVAL;
	}
}

2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
				      size_t len)
{
	struct virtnet_info *vi = netdev_priv(dev);
	int ret;

	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
		return -EOPNOTSUPP;

	ret = snprintf(buf, len, "sby");
	if (ret >= len)
		return -EOPNOTSUPP;

	return 0;
}

2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583
static int virtnet_set_features(struct net_device *dev,
				netdev_features_t features)
{
	struct virtnet_info *vi = netdev_priv(dev);
	u64 offloads;
	int err;

	if ((dev->features ^ features) & NETIF_F_LRO) {
		if (vi->xdp_queue_pairs)
			return -EBUSY;

		if (features & NETIF_F_LRO)
			offloads = vi->guest_offloads_capable;
		else
			offloads = 0;

		err = virtnet_set_guest_offloads(vi, offloads);
		if (err)
			return err;
		vi->guest_offloads = offloads;
	}

	return 0;
}

2584 2585 2586 2587 2588
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
2589
	.ndo_set_mac_address = virtnet_set_mac_address,
2590
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
2591
	.ndo_get_stats64     = virtnet_stats,
2592 2593
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
2594
	.ndo_bpf		= virtnet_xdp,
J
Jason Wang 已提交
2595
	.ndo_xdp_xmit		= virtnet_xdp_xmit,
2596
	.ndo_features_check	= passthru_features_check,
2597
	.ndo_get_phys_port_name	= virtnet_get_phys_port_name,
2598
	.ndo_set_features	= virtnet_set_features,
2599 2600
};

2601
static void virtnet_config_changed_work(struct work_struct *work)
2602
{
2603 2604
	struct virtnet_info *vi =
		container_of(work, struct virtnet_info, config_work);
2605 2606
	u16 v;

2607 2608
	if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
				 struct virtio_net_config, status, &v) < 0)
M
Michael S. Tsirkin 已提交
2609
		return;
2610 2611

	if (v & VIRTIO_NET_S_ANNOUNCE) {
2612
		netdev_notify_peers(vi->dev);
2613 2614
		virtnet_ack_link_announce(vi);
	}
2615 2616 2617 2618 2619

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
M
Michael S. Tsirkin 已提交
2620
		return;
2621 2622 2623 2624

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
2625
		virtnet_update_settings(vi);
2626
		netif_carrier_on(vi->dev);
J
Jason Wang 已提交
2627
		netif_tx_wake_all_queues(vi->dev);
2628 2629
	} else {
		netif_carrier_off(vi->dev);
J
Jason Wang 已提交
2630
		netif_tx_stop_all_queues(vi->dev);
2631 2632 2633 2634 2635 2636 2637
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

2638
	schedule_work(&vi->config_work);
2639 2640
}

J
Jason Wang 已提交
2641 2642
static void virtnet_free_queues(struct virtnet_info *vi)
{
2643 2644
	int i;

2645 2646
	for (i = 0; i < vi->max_queue_pairs; i++) {
		napi_hash_del(&vi->rq[i].napi);
2647
		netif_napi_del(&vi->rq[i].napi);
W
Willem de Bruijn 已提交
2648
		netif_napi_del(&vi->sq[i].napi);
2649
	}
2650

2651 2652 2653 2654 2655
	/* We called napi_hash_del() before netif_napi_del(),
	 * we need to respect an RCU grace period before freeing vi->rq
	 */
	synchronize_net();

J
Jason Wang 已提交
2656 2657
	kfree(vi->rq);
	kfree(vi->sq);
2658
	kfree(vi->ctrl);
J
Jason Wang 已提交
2659 2660
}

2661
static void _free_receive_bufs(struct virtnet_info *vi)
J
Jason Wang 已提交
2662
{
J
John Fastabend 已提交
2663
	struct bpf_prog *old_prog;
J
Jason Wang 已提交
2664 2665 2666 2667 2668
	int i;

	for (i = 0; i < vi->max_queue_pairs; i++) {
		while (vi->rq[i].pages)
			__free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
J
John Fastabend 已提交
2669 2670 2671 2672 2673

		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
		RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
		if (old_prog)
			bpf_prog_put(old_prog);
J
Jason Wang 已提交
2674
	}
2675 2676 2677 2678 2679 2680
}

static void free_receive_bufs(struct virtnet_info *vi)
{
	rtnl_lock();
	_free_receive_bufs(vi);
J
John Fastabend 已提交
2681
	rtnl_unlock();
J
Jason Wang 已提交
2682 2683
}

2684 2685 2686 2687 2688 2689 2690 2691
static void free_receive_page_frags(struct virtnet_info *vi)
{
	int i;
	for (i = 0; i < vi->max_queue_pairs; i++)
		if (vi->rq[i].alloc_frag.page)
			put_page(vi->rq[i].alloc_frag.page);
}

J
Jason Wang 已提交
2692 2693 2694 2695 2696 2697 2698
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
	int i;

	for (i = 0; i < vi->max_queue_pairs; i++) {
		struct virtqueue *vq = vi->sq[i].vq;
J
John Fastabend 已提交
2699
		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2700
			if (!is_xdp_frame(buf))
J
John Fastabend 已提交
2701 2702
				dev_kfree_skb(buf);
			else
2703
				xdp_return_frame(ptr_to_xdp(buf));
J
John Fastabend 已提交
2704
		}
J
Jason Wang 已提交
2705 2706 2707 2708 2709 2710
	}

	for (i = 0; i < vi->max_queue_pairs; i++) {
		struct virtqueue *vq = vi->rq[i].vq;

		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2711
			if (vi->mergeable_rx_bufs) {
2712
				put_page(virt_to_head_page(buf));
2713
			} else if (vi->big_packets) {
2714
				give_pages(&vi->rq[i], buf);
2715
			} else {
2716
				put_page(virt_to_head_page(buf));
2717
			}
J
Jason Wang 已提交
2718 2719 2720 2721
		}
	}
}

2722 2723 2724 2725
static void virtnet_del_vqs(struct virtnet_info *vi)
{
	struct virtio_device *vdev = vi->vdev;

2726
	virtnet_clean_affinity(vi);
J
Jason Wang 已提交
2727

2728
	vdev->config->del_vqs(vdev);
J
Jason Wang 已提交
2729 2730

	virtnet_free_queues(vi);
2731 2732
}

2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744
/* How large should a single buffer be so a queue full of these can fit at
 * least one full packet?
 * Logic below assumes the mergeable buffer header is used.
 */
static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
{
	const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
	unsigned int rq_size = virtqueue_get_vring_size(vq);
	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
	unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
	unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);

2745 2746
	return max(max(min_buf_len, hdr_len) - hdr_len,
		   (unsigned int)GOOD_PACKET_LEN);
2747 2748
}

J
Jason Wang 已提交
2749
static int virtnet_find_vqs(struct virtnet_info *vi)
2750
{
J
Jason Wang 已提交
2751 2752 2753 2754 2755
	vq_callback_t **callbacks;
	struct virtqueue **vqs;
	int ret = -ENOMEM;
	int i, total_vqs;
	const char **names;
2756
	bool *ctx;
J
Jason Wang 已提交
2757 2758 2759 2760 2761 2762 2763 2764 2765

	/* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
	 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
	 * possible control vq.
	 */
	total_vqs = vi->max_queue_pairs * 2 +
		    virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);

	/* Allocate space for find_vqs parameters */
K
Kees Cook 已提交
2766
	vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
J
Jason Wang 已提交
2767 2768
	if (!vqs)
		goto err_vq;
2769
	callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL);
J
Jason Wang 已提交
2770 2771
	if (!callbacks)
		goto err_callback;
2772
	names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL);
J
Jason Wang 已提交
2773 2774
	if (!names)
		goto err_names;
2775
	if (!vi->big_packets || vi->mergeable_rx_bufs) {
K
Kees Cook 已提交
2776
		ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
2777 2778 2779 2780 2781
		if (!ctx)
			goto err_ctx;
	} else {
		ctx = NULL;
	}
J
Jason Wang 已提交
2782 2783 2784 2785 2786 2787

	/* Parameters for control virtqueue, if any */
	if (vi->has_cvq) {
		callbacks[total_vqs - 1] = NULL;
		names[total_vqs - 1] = "control";
	}
2788

J
Jason Wang 已提交
2789 2790 2791 2792 2793 2794 2795 2796
	/* Allocate/initialize parameters for send/receive virtqueues */
	for (i = 0; i < vi->max_queue_pairs; i++) {
		callbacks[rxq2vq(i)] = skb_recv_done;
		callbacks[txq2vq(i)] = skb_xmit_done;
		sprintf(vi->rq[i].name, "input.%d", i);
		sprintf(vi->sq[i].name, "output.%d", i);
		names[rxq2vq(i)] = vi->rq[i].name;
		names[txq2vq(i)] = vi->sq[i].name;
2797 2798
		if (ctx)
			ctx[rxq2vq(i)] = true;
J
Jason Wang 已提交
2799
	}
2800

J
Jason Wang 已提交
2801
	ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
2802
					 names, ctx, NULL);
J
Jason Wang 已提交
2803 2804
	if (ret)
		goto err_find;
2805

J
Jason Wang 已提交
2806 2807
	if (vi->has_cvq) {
		vi->cvq = vqs[total_vqs - 1];
2808
		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2809
			vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2810
	}
J
Jason Wang 已提交
2811 2812 2813

	for (i = 0; i < vi->max_queue_pairs; i++) {
		vi->rq[i].vq = vqs[rxq2vq(i)];
2814
		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
J
Jason Wang 已提交
2815 2816 2817
		vi->sq[i].vq = vqs[txq2vq(i)];
	}

2818
	/* run here: ret == 0. */
J
Jason Wang 已提交
2819 2820 2821


err_find:
2822 2823
	kfree(ctx);
err_ctx:
J
Jason Wang 已提交
2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836
	kfree(names);
err_names:
	kfree(callbacks);
err_callback:
	kfree(vqs);
err_vq:
	return ret;
}

static int virtnet_alloc_queues(struct virtnet_info *vi)
{
	int i;

2837 2838 2839
	vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
	if (!vi->ctrl)
		goto err_ctrl;
K
Kees Cook 已提交
2840
	vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);
J
Jason Wang 已提交
2841 2842
	if (!vi->sq)
		goto err_sq;
K
Kees Cook 已提交
2843
	vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);
2844
	if (!vi->rq)
J
Jason Wang 已提交
2845 2846 2847 2848 2849 2850 2851
		goto err_rq;

	INIT_DELAYED_WORK(&vi->refill, refill_work);
	for (i = 0; i < vi->max_queue_pairs; i++) {
		vi->rq[i].pages = NULL;
		netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
			       napi_weight);
2852 2853
		netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
				  napi_tx ? napi_weight : 0);
J
Jason Wang 已提交
2854 2855

		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
J
Johannes Berg 已提交
2856
		ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
J
Jason Wang 已提交
2857
		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
T
Toshiaki Makita 已提交
2858 2859 2860

		u64_stats_init(&vi->rq[i].stats.syncp);
		u64_stats_init(&vi->sq[i].stats.syncp);
J
Jason Wang 已提交
2861 2862 2863 2864 2865 2866 2867
	}

	return 0;

err_rq:
	kfree(vi->sq);
err_sq:
2868 2869
	kfree(vi->ctrl);
err_ctrl:
J
Jason Wang 已提交
2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
	return -ENOMEM;
}

static int init_vqs(struct virtnet_info *vi)
{
	int ret;

	/* Allocate send & receive queues */
	ret = virtnet_alloc_queues(vi);
	if (ret)
		goto err;

	ret = virtnet_find_vqs(vi);
	if (ret)
		goto err_free;

2886
	get_online_cpus();
2887
	virtnet_set_affinity(vi);
2888 2889
	put_online_cpus();

J
Jason Wang 已提交
2890 2891 2892 2893 2894 2895
	return 0;

err_free:
	virtnet_free_queues(vi);
err:
	return ret;
2896 2897
}

2898 2899
#ifdef CONFIG_SYSFS
static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
2900
		char *buf)
2901 2902 2903
{
	struct virtnet_info *vi = netdev_priv(queue->dev);
	unsigned int queue_index = get_netdev_rx_queue_index(queue);
2904 2905
	unsigned int headroom = virtnet_get_headroom(vi);
	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
J
Johannes Berg 已提交
2906
	struct ewma_pkt_len *avg;
2907 2908 2909

	BUG_ON(queue_index >= vi->max_queue_pairs);
	avg = &vi->rq[queue_index].mrg_avg_pkt_len;
2910
	return sprintf(buf, "%u\n",
2911 2912
		       get_mergeable_buf_len(&vi->rq[queue_index], avg,
				       SKB_DATA_ALIGN(headroom + tailroom)));
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928
}

static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
	__ATTR_RO(mergeable_rx_buffer_size);

static struct attribute *virtio_net_mrg_rx_attrs[] = {
	&mergeable_rx_buffer_size_attribute.attr,
	NULL
};

static const struct attribute_group virtio_net_mrg_rx_group = {
	.name = "virtio_net",
	.attrs = virtio_net_mrg_rx_attrs
};
#endif

2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962
static bool virtnet_fail_on_feature(struct virtio_device *vdev,
				    unsigned int fbit,
				    const char *fname, const char *dname)
{
	if (!virtio_has_feature(vdev, fbit))
		return false;

	dev_err(&vdev->dev, "device advertises feature %s but not %s",
		fname, dname);

	return true;
}

#define VIRTNET_FAIL_ON(vdev, fbit, dbit)			\
	virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)

static bool virtnet_validate_features(struct virtio_device *vdev)
{
	if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
	    (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
			     "VIRTIO_NET_F_CTRL_VQ") ||
	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
			     "VIRTIO_NET_F_CTRL_VQ") ||
	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
			     "VIRTIO_NET_F_CTRL_VQ") ||
	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
			     "VIRTIO_NET_F_CTRL_VQ"))) {
		return false;
	}

	return true;
}

2963 2964 2965
#define MIN_MTU ETH_MIN_MTU
#define MAX_MTU ETH_MAX_MTU

2966
static int virtnet_validate(struct virtio_device *vdev)
R
Rusty Russell 已提交
2967
{
2968 2969 2970 2971 2972 2973
	if (!vdev->config->get) {
		dev_err(&vdev->dev, "%s failure: config access disabled\n",
			__func__);
		return -EINVAL;
	}

2974 2975 2976
	if (!virtnet_validate_features(vdev))
		return -EINVAL;

2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
		int mtu = virtio_cread16(vdev,
					 offsetof(struct virtio_net_config,
						  mtu));
		if (mtu < MIN_MTU)
			__virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
	}

	return 0;
}

static int virtnet_probe(struct virtio_device *vdev)
{
T
Toshiaki Makita 已提交
2990
	int i, err = -ENOMEM;
2991 2992 2993 2994 2995
	struct net_device *dev;
	struct virtnet_info *vi;
	u16 max_queue_pairs;
	int mtu;

J
Jason Wang 已提交
2996
	/* Find if host supports multiqueue virtio_net device */
2997 2998 2999
	err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
				   struct virtio_net_config,
				   max_virtqueue_pairs, &max_queue_pairs);
J
Jason Wang 已提交
3000 3001 3002 3003 3004 3005

	/* We need at least 2 queue's */
	if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
	    max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
	    !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
		max_queue_pairs = 1;
R
Rusty Russell 已提交
3006 3007

	/* Allocate ourselves a network device with room for our info */
J
Jason Wang 已提交
3008
	dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
R
Rusty Russell 已提交
3009 3010 3011 3012
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
3013
	dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
3014
	dev->netdev_ops = &virtnet_netdev;
R
Rusty Russell 已提交
3015
	dev->features = NETIF_F_HIGHDMA;
3016

3017
	dev->ethtool_ops = &virtnet_ethtool_ops;
R
Rusty Russell 已提交
3018 3019 3020
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
3021
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
R
Rusty Russell 已提交
3022
		/* This opens up the world of extra features. */
J
Jason Wang 已提交
3023
		dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3024
		if (csum)
J
Jason Wang 已提交
3025
			dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3026 3027

		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
3028
			dev->hw_features |= NETIF_F_TSO
R
Rusty Russell 已提交
3029 3030
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
3031
		/* Individual feature bits: what can host handle? */
3032 3033 3034 3035 3036 3037 3038
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
			dev->hw_features |= NETIF_F_TSO;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
			dev->hw_features |= NETIF_F_TSO6;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
			dev->hw_features |= NETIF_F_TSO_ECN;

3039 3040
		dev->features |= NETIF_F_GSO_ROBUST;

3041
		if (gso)
3042
			dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
3043
		/* (!csum && gso) case will be fixed by register_netdev() */
R
Rusty Russell 已提交
3044
	}
3045 3046
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
		dev->features |= NETIF_F_RXCSUM;
3047 3048 3049 3050 3051
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6))
		dev->features |= NETIF_F_LRO;
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))
		dev->hw_features |= NETIF_F_LRO;
R
Rusty Russell 已提交
3052

3053 3054
	dev->vlan_features = dev->features;

3055 3056 3057 3058
	/* MTU range: 68 - 65535 */
	dev->min_mtu = MIN_MTU;
	dev->max_mtu = MAX_MTU;

R
Rusty Russell 已提交
3059
	/* Configuration may specify what MAC to use.  Otherwise random. */
3060 3061 3062 3063 3064
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		virtio_cread_bytes(vdev,
				   offsetof(struct virtio_net_config, mac),
				   dev->dev_addr, dev->addr_len);
	else
3065
		eth_hw_addr_random(dev);
R
Rusty Russell 已提交
3066 3067 3068 3069 3070

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
	vi->dev = dev;
	vi->vdev = vdev;
3071
	vdev->priv = vi;
3072

3073
	INIT_WORK(&vi->config_work, virtnet_config_changed_work);
R
Rusty Russell 已提交
3074

3075
	/* If we can receive ANY GSO packets, we must allocate large ones. */
3076 3077
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
3078 3079
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
3080 3081
		vi->big_packets = true;

3082 3083 3084
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

3085 3086
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
	    virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3087 3088 3089 3090
		vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
	else
		vi->hdr_len = sizeof(struct virtio_net_hdr);

3091 3092
	if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
	    virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3093 3094
		vi->any_header_sg = true;

J
Jason Wang 已提交
3095 3096 3097
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
		vi->has_cvq = true;

3098 3099 3100 3101
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
		mtu = virtio_cread16(vdev,
				     offsetof(struct virtio_net_config,
					      mtu));
3102
		if (mtu < dev->min_mtu) {
3103 3104 3105
			/* Should never trigger: MTU was previously validated
			 * in virtnet_validate.
			 */
3106 3107 3108
			dev_err(&vdev->dev,
				"device MTU appears to have changed it is now %d < %d",
				mtu, dev->min_mtu);
T
Toshiaki Makita 已提交
3109
			goto free;
3110
		}
3111

3112 3113 3114
		dev->mtu = mtu;
		dev->max_mtu = mtu;

3115 3116 3117
		/* TODO: size buffers correctly in this case. */
		if (dev->mtu > ETH_DATA_LEN)
			vi->big_packets = true;
3118 3119
	}

3120 3121
	if (vi->any_header_sg)
		dev->needed_headroom = vi->hdr_len;
3122

3123 3124 3125 3126 3127
	/* Enable multiqueue by default */
	if (num_online_cpus() >= max_queue_pairs)
		vi->curr_queue_pairs = max_queue_pairs;
	else
		vi->curr_queue_pairs = num_online_cpus();
J
Jason Wang 已提交
3128 3129 3130
	vi->max_queue_pairs = max_queue_pairs;

	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */
3131
	err = init_vqs(vi);
3132
	if (err)
T
Toshiaki Makita 已提交
3133
		goto free;
R
Rusty Russell 已提交
3134

3135 3136 3137 3138
#ifdef CONFIG_SYSFS
	if (vi->mergeable_rx_bufs)
		dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
#endif
3139 3140
	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
J
Jason Wang 已提交
3141

3142 3143
	virtnet_init_settings(dev);

3144 3145
	if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
		vi->failover = net_failover_create(vi->dev);
3146 3147
		if (IS_ERR(vi->failover)) {
			err = PTR_ERR(vi->failover);
3148
			goto free_vqs;
3149
		}
3150 3151
	}

R
Rusty Russell 已提交
3152 3153 3154
	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
3155
		goto free_failover;
R
Rusty Russell 已提交
3156
	}
3157

M
Michael S. Tsirkin 已提交
3158 3159
	virtio_device_ready(vdev);

3160
	err = virtnet_cpu_notif_add(vi);
3161 3162
	if (err) {
		pr_debug("virtio_net: registering cpu notifier failed\n");
3163
		goto free_unregister_netdev;
3164 3165
	}

3166
	virtnet_set_queues(vi, vi->curr_queue_pairs);
3167

J
Jason Wang 已提交
3168 3169
	/* Assume link up if device can't report link status,
	   otherwise get link status from config. */
3170
	netif_carrier_off(dev);
J
Jason Wang 已提交
3171
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
3172
		schedule_work(&vi->config_work);
J
Jason Wang 已提交
3173 3174
	} else {
		vi->status = VIRTIO_NET_S_LINK_UP;
3175
		virtnet_update_settings(vi);
J
Jason Wang 已提交
3176 3177
		netif_carrier_on(dev);
	}
3178

3179 3180 3181
	for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
		if (virtio_has_feature(vi->vdev, guest_offloads[i]))
			set_bit(guest_offloads[i], &vi->guest_offloads);
3182
	vi->guest_offloads_capable = vi->guest_offloads;
3183

J
Jason Wang 已提交
3184 3185 3186
	pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
		 dev->name, max_queue_pairs);

R
Rusty Russell 已提交
3187 3188
	return 0;

3189
free_unregister_netdev:
3190 3191
	vi->vdev->config->reset(vdev);

3192
	unregister_netdev(dev);
3193 3194
free_failover:
	net_failover_destroy(vi->failover);
3195
free_vqs:
J
Jason Wang 已提交
3196
	cancel_delayed_work_sync(&vi->refill);
3197
	free_receive_page_frags(vi);
3198
	virtnet_del_vqs(vi);
R
Rusty Russell 已提交
3199 3200 3201 3202 3203
free:
	free_netdev(dev);
	return err;
}

3204
static void remove_vq_common(struct virtnet_info *vi)
R
Rusty Russell 已提交
3205
{
3206
	vi->vdev->config->reset(vi->vdev);
S
Shirley Ma 已提交
3207 3208

	/* Free unused buffers in both send and recv, if any. */
3209
	free_unused_bufs(vi);
3210

J
Jason Wang 已提交
3211
	free_receive_bufs(vi);
3212

3213 3214
	free_receive_page_frags(vi);

J
Jason Wang 已提交
3215
	virtnet_del_vqs(vi);
3216 3217
}

3218
static void virtnet_remove(struct virtio_device *vdev)
3219 3220 3221
{
	struct virtnet_info *vi = vdev->priv;

3222
	virtnet_cpu_notif_remove(vi);
3223

3224 3225
	/* Make sure no work handler is accessing the device. */
	flush_work(&vi->config_work);
3226

3227 3228
	unregister_netdev(vi->dev);

3229 3230
	net_failover_destroy(vi->failover);

3231
	remove_vq_common(vi);
3232

3233
	free_netdev(vi->dev);
R
Rusty Russell 已提交
3234 3235
}

3236
static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
3237 3238 3239
{
	struct virtnet_info *vi = vdev->priv;

3240
	virtnet_cpu_notif_remove(vi);
3241
	virtnet_freeze_down(vdev);
3242 3243 3244 3245 3246
	remove_vq_common(vi);

	return 0;
}

3247
static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
3248 3249
{
	struct virtnet_info *vi = vdev->priv;
3250
	int err;
3251

3252
	err = virtnet_restore_up(vdev);
3253 3254
	if (err)
		return err;
J
Jason Wang 已提交
3255 3256
	virtnet_set_queues(vi, vi->curr_queue_pairs);

3257
	err = virtnet_cpu_notif_add(vi);
3258 3259 3260
	if (err)
		return err;

3261 3262 3263
	return 0;
}

R
Rusty Russell 已提交
3264 3265 3266 3267 3268
static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
#define VIRTNET_FEATURES \
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
	VIRTIO_NET_F_MAC, \
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
	VIRTIO_NET_F_CTRL_MAC_ADDR, \
3279
	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3280
	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
3281

3282
static unsigned int features[] = {
3283 3284 3285 3286 3287 3288
	VIRTNET_FEATURES,
};

static unsigned int features_legacy[] = {
	VIRTNET_FEATURES,
	VIRTIO_NET_F_GSO,
3289
	VIRTIO_F_ANY_LAYOUT,
3290 3291
};

3292
static struct virtio_driver virtio_net_driver = {
3293 3294
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
3295 3296
	.feature_table_legacy = features_legacy,
	.feature_table_size_legacy = ARRAY_SIZE(features_legacy),
R
Rusty Russell 已提交
3297 3298 3299
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
3300
	.validate =	virtnet_validate,
R
Rusty Russell 已提交
3301
	.probe =	virtnet_probe,
3302
	.remove =	virtnet_remove,
3303
	.config_changed = virtnet_config_changed,
3304
#ifdef CONFIG_PM_SLEEP
3305 3306 3307
	.freeze =	virtnet_freeze,
	.restore =	virtnet_restore,
#endif
R
Rusty Russell 已提交
3308 3309
};

3310 3311 3312 3313
static __init int virtio_net_driver_init(void)
{
	int ret;

T
Thomas Gleixner 已提交
3314
	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
3315 3316 3317 3318 3319
				      virtnet_cpu_online,
				      virtnet_cpu_down_prep);
	if (ret < 0)
		goto out;
	virtionet_online = ret;
T
Thomas Gleixner 已提交
3320
	ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339
				      NULL, virtnet_cpu_dead);
	if (ret)
		goto err_dead;

        ret = register_virtio_driver(&virtio_net_driver);
	if (ret)
		goto err_virtio;
	return 0;
err_virtio:
	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
err_dead:
	cpuhp_remove_multi_state(virtionet_online);
out:
	return ret;
}
module_init(virtio_net_driver_init);

static __exit void virtio_net_driver_exit(void)
{
A
Andrew Jones 已提交
3340
	unregister_virtio_driver(&virtio_net_driver);
3341 3342 3343 3344
	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
	cpuhp_remove_multi_state(virtionet_online);
}
module_exit(virtio_net_driver_exit);
R
Rusty Russell 已提交
3345 3346 3347 3348

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");