virtio_net.c 31.6 KB
Newer Older
1
/* A network driver using virtio.
R
Rusty Russell 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
22
#include <linux/ethtool.h>
R
Rusty Russell 已提交
23 24 25 26
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/scatterlist.h>
27
#include <linux/if_vlan.h>
28
#include <linux/slab.h>
R
Rusty Russell 已提交
29

30 31 32
static int napi_weight = 128;
module_param(napi_weight, int, 0444);

33
static bool csum = true, gso = true;
R
Rusty Russell 已提交
34 35 36
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);

R
Rusty Russell 已提交
37
/* FIXME: MTU in config. */
38
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
39
#define GOOD_COPY_LEN	128
R
Rusty Russell 已提交
40

41
#define VIRTNET_SEND_COMMAND_SG_MAX    2
42
#define VIRTNET_DRIVER_VERSION "1.0.0"
43

44 45 46 47 48 49 50 51 52
struct virtnet_stats {
	struct u64_stats_sync syncp;
	u64 tx_bytes;
	u64 tx_packets;

	u64 rx_bytes;
	u64 rx_packets;
};

53
struct virtnet_info {
R
Rusty Russell 已提交
54
	struct virtio_device *vdev;
55
	struct virtqueue *rvq, *svq, *cvq;
R
Rusty Russell 已提交
56 57
	struct net_device *dev;
	struct napi_struct napi;
58
	unsigned int status;
R
Rusty Russell 已提交
59 60 61 62

	/* Number of input buffers, and max we've ever had. */
	unsigned int num, max;

63 64 65
	/* I like... big packets and I cannot lie! */
	bool big_packets;

66 67 68
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

69 70 71
	/* Active statistics */
	struct virtnet_stats __percpu *stats;

72 73 74
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

75 76
	/* Chain pages by the private ptr. */
	struct page *pages;
77 78 79 80

	/* fragments + linear part + virtio header */
	struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
	struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
R
Rusty Russell 已提交
81 82
};

83 84 85 86 87
struct skb_vnet_hdr {
	union {
		struct virtio_net_hdr hdr;
		struct virtio_net_hdr_mrg_rxbuf mhdr;
	};
88
	unsigned int num_sg;
89 90
};

91 92 93 94 95 96 97 98 99 100
struct padded_vnet_hdr {
	struct virtio_net_hdr hdr;
	/*
	 * virtio_net_hdr should be in a separated sg buffer because of a
	 * QEMU bug, and data sg buffer shares same page with this header sg.
	 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
	 */
	char padding[6];
};

101
static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
R
Rusty Russell 已提交
102
{
103
	return (struct skb_vnet_hdr *)skb->cb;
R
Rusty Russell 已提交
104 105
}

106 107 108 109 110
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
static void give_pages(struct virtnet_info *vi, struct page *page)
111
{
112
	struct page *end;
113

114 115 116 117
	/* Find end of list, sew whole thing into vi->pages. */
	for (end = page; end->private; end = (struct page *)end->private);
	end->private = (unsigned long)vi->pages;
	vi->pages = page;
118 119
}

120 121 122 123
static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
{
	struct page *p = vi->pages;

124
	if (p) {
125
		vi->pages = (struct page *)p->private;
126 127 128
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
129 130 131 132
		p = alloc_page(gfp_mask);
	return p;
}

133
static void skb_xmit_done(struct virtqueue *svq)
R
Rusty Russell 已提交
134
{
135
	struct virtnet_info *vi = svq->vdev->priv;
R
Rusty Russell 已提交
136

137
	/* Suppress further interrupts. */
138
	virtqueue_disable_cb(svq);
139

140
	/* We were probably waiting for more output buffers. */
R
Rusty Russell 已提交
141 142 143
	netif_wake_queue(vi->dev);
}

144 145
static void set_skb_frag(struct sk_buff *skb, struct page *page,
			 unsigned int offset, unsigned int *len)
R
Rusty Russell 已提交
146
{
147
	int size = min((unsigned)PAGE_SIZE - offset, *len);
148 149
	int i = skb_shinfo(skb)->nr_frags;

150
	__skb_fill_page_desc(skb, i, page, offset, size);
151

152 153
	skb->data_len += size;
	skb->len += size;
154
	skb->truesize += PAGE_SIZE;
155
	skb_shinfo(skb)->nr_frags++;
156
	*len -= size;
157
}
158

159
/* Called from bottom half context */
160 161 162 163 164 165 166
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct page *page, unsigned int len)
{
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
	unsigned int copy, hdr_len, offset;
	char *p;
167

168
	p = page_address(page);
169

170 171 172 173
	/* copy small packet so we can reuse these pages for small data */
	skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;
174

175
	hdr = skb_vnet_hdr(skb);
176

177 178 179 180 181 182 183
	if (vi->mergeable_rx_bufs) {
		hdr_len = sizeof hdr->mhdr;
		offset = hdr_len;
	} else {
		hdr_len = sizeof hdr->hdr;
		offset = sizeof(struct padded_vnet_hdr);
	}
184

185
	memcpy(hdr, p, hdr_len);
186

187 188
	len -= hdr_len;
	p += offset;
189

190 191 192 193
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	memcpy(skb_put(skb, copy), p, copy);
194

195 196
	len -= copy;
	offset += copy;
197

198 199 200 201 202 203 204 205 206 207 208 209 210
	/*
	 * Verify that we can indeed put this data into a skb.
	 * This is here to handle cases when the device erroneously
	 * tries to receive more than is possible. This is usually
	 * the case of a broken device.
	 */
	if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
		if (net_ratelimit())
			pr_debug("%s: too much data\n", skb->dev->name);
		dev_kfree_skb(skb);
		return NULL;
	}

211 212 213 214 215
	while (len) {
		set_skb_frag(skb, page, offset, &len);
		page = (struct page *)page->private;
		offset = 0;
	}
216

217 218
	if (page)
		give_pages(vi, page);
219

220 221
	return skb;
}
222

223 224 225 226 227 228 229 230 231 232 233 234 235 236
static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
{
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
	struct page *page;
	int num_buf, i, len;

	num_buf = hdr->mhdr.num_buffers;
	while (--num_buf) {
		i = skb_shinfo(skb)->nr_frags;
		if (i >= MAX_SKB_FRAGS) {
			pr_debug("%s: packet too long\n", skb->dev->name);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
		}
237
		page = virtqueue_get_buf(vi->rvq, &len);
238 239 240 241 242
		if (!page) {
			pr_debug("%s: rx error: %d buffers missing\n",
				 skb->dev->name, hdr->mhdr.num_buffers);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
243
		}
244

245 246 247 248 249 250 251 252 253 254 255 256 257
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;

		set_skb_frag(skb, page, 0, &len);

		--vi->num;
	}
	return 0;
}

static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
{
	struct virtnet_info *vi = netdev_priv(dev);
258
	struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
259 260 261
	struct sk_buff *skb;
	struct page *page;
	struct skb_vnet_hdr *hdr;
262

263 264 265 266 267 268 269 270 271
	if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		return;
	}
272

273 274 275 276 277 278 279 280
	if (!vi->mergeable_rx_bufs && !vi->big_packets) {
		skb = buf;
		len -= sizeof(struct virtio_net_hdr);
		skb_trim(skb, len);
	} else {
		page = buf;
		skb = page_to_skb(vi, page, len);
		if (unlikely(!skb)) {
281
			dev->stats.rx_dropped++;
282 283
			give_pages(vi, page);
			return;
284
		}
285 286 287 288 289
		if (vi->mergeable_rx_bufs)
			if (receive_mergeable(vi, skb)) {
				dev_kfree_skb(skb);
				return;
			}
290
	}
291

292
	hdr = skb_vnet_hdr(skb);
293 294 295 296 297

	u64_stats_update_begin(&stats->syncp);
	stats->rx_bytes += skb->len;
	stats->rx_packets++;
	u64_stats_update_end(&stats->syncp);
R
Rusty Russell 已提交
298

299
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
R
Rusty Russell 已提交
300
		pr_debug("Needs csum!\n");
301 302 303
		if (!skb_partial_csum_set(skb,
					  hdr->hdr.csum_start,
					  hdr->hdr.csum_offset))
R
Rusty Russell 已提交
304
			goto frame_err;
305 306
	} else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
		skb->ip_summed = CHECKSUM_UNNECESSARY;
R
Rusty Russell 已提交
307 308
	}

309 310 311 312
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

313
	if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
R
Rusty Russell 已提交
314
		pr_debug("GSO!\n");
315
		switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
R
Rusty Russell 已提交
316 317 318 319 320 321 322 323 324 325 326 327
		case VIRTIO_NET_HDR_GSO_TCPV4:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
			break;
		case VIRTIO_NET_HDR_GSO_UDP:
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
			break;
		case VIRTIO_NET_HDR_GSO_TCPV6:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
			break;
		default:
			if (net_ratelimit())
				printk(KERN_WARNING "%s: bad gso type %u.\n",
328
				       dev->name, hdr->hdr.gso_type);
R
Rusty Russell 已提交
329 330 331
			goto frame_err;
		}

332
		if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
R
Rusty Russell 已提交
333 334
			skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;

335
		skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
R
Rusty Russell 已提交
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
		if (skb_shinfo(skb)->gso_size == 0) {
			if (net_ratelimit())
				printk(KERN_WARNING "%s: zero gso size.\n",
				       dev->name);
			goto frame_err;
		}

		/* Header must be checked, and gso_segs computed. */
		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
		skb_shinfo(skb)->gso_segs = 0;
	}

	netif_receive_skb(skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

356
static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
R
Rusty Russell 已提交
357 358
{
	struct sk_buff *skb;
359 360
	struct skb_vnet_hdr *hdr;
	int err;
361

362
	skb = __netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN, gfp);
363 364
	if (unlikely(!skb))
		return -ENOMEM;
R
Rusty Russell 已提交
365

366
	skb_put(skb, MAX_PACKET_LEN);
367

368
	hdr = skb_vnet_hdr(skb);
369
	sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
370

371
	skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
372

373
	err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, 2, skb, gfp);
374 375
	if (err < 0)
		dev_kfree_skb(skb);
376

377 378
	return err;
}
379

380 381 382 383 384 385
static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

386
	/* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
387 388 389 390 391 392
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
		first = get_a_page(vi, gfp);
		if (!first) {
			if (list)
				give_pages(vi, list);
			return -ENOMEM;
393
		}
394
		sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
395

396 397 398 399
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
R
Rusty Russell 已提交
400

401 402 403 404 405 406 407
	first = get_a_page(vi, gfp);
	if (!first) {
		give_pages(vi, list);
		return -ENOMEM;
	}
	p = page_address(first);

408 409 410
	/* vi->rx_sg[0], vi->rx_sg[1] share the same page */
	/* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
	sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
411

412
	/* vi->rx_sg[1] for data packet, from offset */
413
	offset = sizeof(struct padded_vnet_hdr);
414
	sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
415 416 417

	/* chain first in list head */
	first->private = (unsigned long)list;
418 419
	err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
				first, gfp);
420 421 422 423
	if (err < 0)
		give_pages(vi, first);

	return err;
R
Rusty Russell 已提交
424 425
}

426
static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
427
{
428
	struct page *page;
429 430
	int err;

431 432 433
	page = get_a_page(vi, gfp);
	if (!page)
		return -ENOMEM;
434

435
	sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
436

437
	err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, 1, page, gfp);
438 439
	if (err < 0)
		give_pages(vi, page);
440

441 442
	return err;
}
443

444 445 446 447 448 449 450
/*
 * Returns false if we couldn't fill entirely (OOM).
 *
 * Normally run in the receive path, but can also be run from ndo_open
 * before we're receiving packets, or from refill_work which is
 * careful to disable receiving (using napi_disable).
 */
451 452 453
static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
{
	int err;
454
	bool oom;
455

456 457 458 459 460 461 462
	do {
		if (vi->mergeable_rx_bufs)
			err = add_recvbuf_mergeable(vi, gfp);
		else if (vi->big_packets)
			err = add_recvbuf_big(vi, gfp);
		else
			err = add_recvbuf_small(vi, gfp);
463

464 465
		oom = err == -ENOMEM;
		if (err < 0)
466
			break;
467
		++vi->num;
468
	} while (err > 0);
469 470
	if (unlikely(vi->num > vi->max))
		vi->max = vi->num;
471
	virtqueue_kick(vi->rvq);
472
	return !oom;
473 474
}

475
static void skb_recv_done(struct virtqueue *rvq)
R
Rusty Russell 已提交
476 477
{
	struct virtnet_info *vi = rvq->vdev->priv;
478
	/* Schedule NAPI, Suppress further interrupts if successful. */
479
	if (napi_schedule_prep(&vi->napi)) {
480
		virtqueue_disable_cb(rvq);
481
		__napi_schedule(&vi->napi);
482
	}
R
Rusty Russell 已提交
483 484
}

485 486 487 488 489 490 491 492 493 494 495 496 497 498
static void virtnet_napi_enable(struct virtnet_info *vi)
{
	napi_enable(&vi->napi);

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets
	 * now.  virtnet_poll wants re-enable the queue, so we disable here.
	 * We synchronize against interrupts via NAPI_STATE_SCHED */
	if (napi_schedule_prep(&vi->napi)) {
		virtqueue_disable_cb(vi->rvq);
		__napi_schedule(&vi->napi);
	}
}

499 500 501 502 503 504 505
static void refill_work(struct work_struct *work)
{
	struct virtnet_info *vi;
	bool still_empty;

	vi = container_of(work, struct virtnet_info, refill.work);
	napi_disable(&vi->napi);
506
	still_empty = !try_fill_recv(vi, GFP_KERNEL);
507
	virtnet_napi_enable(vi);
508 509 510 511

	/* In theory, this can happen: if we don't get any buffers in
	 * we will *never* try to fill again. */
	if (still_empty)
512
		queue_delayed_work(system_nrt_wq, &vi->refill, HZ/2);
513 514
}

R
Rusty Russell 已提交
515 516 517
static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
518
	void *buf;
R
Rusty Russell 已提交
519 520 521 522
	unsigned int len, received = 0;

again:
	while (received < budget &&
523
	       (buf = virtqueue_get_buf(vi->rvq, &len)) != NULL) {
524 525
		receive_buf(vi->dev, buf, len);
		--vi->num;
R
Rusty Russell 已提交
526 527 528
		received++;
	}

529 530
	if (vi->num < vi->max / 2) {
		if (!try_fill_recv(vi, GFP_ATOMIC))
531
			queue_delayed_work(system_nrt_wq, &vi->refill, 0);
532
	}
R
Rusty Russell 已提交
533

534 535
	/* Out of packets? */
	if (received < budget) {
536
		napi_complete(napi);
537
		if (unlikely(!virtqueue_enable_cb(vi->rvq)) &&
538
		    napi_schedule_prep(napi)) {
539
			virtqueue_disable_cb(vi->rvq);
540
			__napi_schedule(napi);
R
Rusty Russell 已提交
541
			goto again;
542
		}
R
Rusty Russell 已提交
543 544 545 546 547
	}

	return received;
}

548
static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
R
Rusty Russell 已提交
549 550
{
	struct sk_buff *skb;
551
	unsigned int len, tot_sgs = 0;
552
	struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
R
Rusty Russell 已提交
553

554
	while ((skb = virtqueue_get_buf(vi->svq, &len)) != NULL) {
R
Rusty Russell 已提交
555
		pr_debug("Sent skb %p\n", skb);
556 557 558 559 560 561

		u64_stats_update_begin(&stats->syncp);
		stats->tx_bytes += skb->len;
		stats->tx_packets++;
		u64_stats_update_end(&stats->syncp);

562
		tot_sgs += skb_vnet_hdr(skb)->num_sg;
563
		dev_kfree_skb_any(skb);
R
Rusty Russell 已提交
564
	}
565
	return tot_sgs;
R
Rusty Russell 已提交
566 567
}

568
static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
R
Rusty Russell 已提交
569
{
570
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
R
Rusty Russell 已提交
571 572
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;

J
Johannes Berg 已提交
573
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
R
Rusty Russell 已提交
574 575

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
576
		hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
577
		hdr->hdr.csum_start = skb_checksum_start_offset(skb);
578
		hdr->hdr.csum_offset = skb->csum_offset;
R
Rusty Russell 已提交
579
	} else {
580 581
		hdr->hdr.flags = 0;
		hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
R
Rusty Russell 已提交
582 583 584
	}

	if (skb_is_gso(skb)) {
585 586
		hdr->hdr.hdr_len = skb_headlen(skb);
		hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
R
Rusty Russell 已提交
587
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
588
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
R
Rusty Russell 已提交
589
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
590
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
R
Rusty Russell 已提交
591
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
592
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
R
Rusty Russell 已提交
593 594
		else
			BUG();
R
Rusty Russell 已提交
595
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
596
			hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
R
Rusty Russell 已提交
597
	} else {
598 599
		hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
		hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
R
Rusty Russell 已提交
600 601
	}

602
	hdr->mhdr.num_buffers = 0;
603 604 605

	/* Encode metadata header at front. */
	if (vi->mergeable_rx_bufs)
606
		sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
607
	else
608
		sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
609

610
	hdr->num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
611
	return virtqueue_add_buf(vi->svq, vi->tx_sg, hdr->num_sg,
612
				 0, skb, GFP_ATOMIC);
613 614
}

615
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
616 617
{
	struct virtnet_info *vi = netdev_priv(dev);
618
	int capacity;
619 620 621

	/* Free up any pending old buffers before queueing new ones. */
	free_old_xmit_skbs(vi);
622

623
	/* Try to transmit */
624 625 626 627
	capacity = xmit_skb(vi, skb);

	/* This can happen with OOM and indirect buffers. */
	if (unlikely(capacity < 0)) {
628 629 630 631 632 633 634 635 636 637
		if (net_ratelimit()) {
			if (likely(capacity == -ENOMEM)) {
				dev_warn(&dev->dev,
					 "TX queue failure: out of memory\n");
			} else {
				dev->stats.tx_fifo_errors++;
				dev_warn(&dev->dev,
					 "Unexpected TX queue failure: %d\n",
					 capacity);
			}
638
		}
639 640 641
		dev->stats.tx_dropped++;
		kfree_skb(skb);
		return NETDEV_TX_OK;
R
Rusty Russell 已提交
642
	}
643
	virtqueue_kick(vi->svq);
644

645 646 647 648 649 650 651 652
	/* Don't wait up for transmitted skbs to be freed. */
	skb_orphan(skb);
	nf_reset(skb);

	/* Apparently nice girls don't return TX_BUSY; stop the queue
	 * before it gets out of hand.  Naturally, this wastes entries. */
	if (capacity < 2+MAX_SKB_FRAGS) {
		netif_stop_queue(dev);
653
		if (unlikely(!virtqueue_enable_cb_delayed(vi->svq))) {
654 655 656 657
			/* More just got used, free them then recheck. */
			capacity += free_old_xmit_skbs(vi);
			if (capacity >= 2+MAX_SKB_FRAGS) {
				netif_start_queue(dev);
658
				virtqueue_disable_cb(vi->svq);
659 660
			}
		}
661
	}
662 663

	return NETDEV_TX_OK;
R
Rusty Russell 已提交
664 665
}

666 667 668 669 670 671 672 673 674 675
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
	int ret;

	ret = eth_mac_addr(dev, p);
	if (ret)
		return ret;

676 677 678
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
		                  dev->dev_addr, dev->addr_len);
679 680 681 682

	return 0;
}

683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
					       struct rtnl_link_stats64 *tot)
{
	struct virtnet_info *vi = netdev_priv(dev);
	int cpu;
	unsigned int start;

	for_each_possible_cpu(cpu) {
		struct virtnet_stats __percpu *stats
			= per_cpu_ptr(vi->stats, cpu);
		u64 tpackets, tbytes, rpackets, rbytes;

		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			tpackets = stats->tx_packets;
			tbytes   = stats->tx_bytes;
			rpackets = stats->rx_packets;
			rbytes   = stats->rx_bytes;
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		tot->rx_packets += rpackets;
		tot->tx_packets += tpackets;
		tot->rx_bytes   += rbytes;
		tot->tx_bytes   += tbytes;
	}

	tot->tx_dropped = dev->stats.tx_dropped;
710
	tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
711 712 713 714 715 716 717
	tot->rx_dropped = dev->stats.rx_dropped;
	tot->rx_length_errors = dev->stats.rx_length_errors;
	tot->rx_frame_errors = dev->stats.rx_frame_errors;

	return tot;
}

718 719 720 721 722 723 724 725 726
#ifdef CONFIG_NET_POLL_CONTROLLER
static void virtnet_netpoll(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_schedule(&vi->napi);
}
#endif

R
Rusty Russell 已提交
727 728 729 730
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

731 732
	/* Make sure we have some buffers: if oom use wq. */
	if (!try_fill_recv(vi, GFP_KERNEL))
733
		queue_delayed_work(system_nrt_wq, &vi->refill, 0);
734

735
	virtnet_napi_enable(vi);
R
Rusty Russell 已提交
736 737 738
	return 0;
}

739 740 741 742 743 744 745 746
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
 * never fail unless improperly formated.
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
				 struct scatterlist *data, int out, int in)
{
747
	struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
748 749 750
	struct virtio_net_ctrl_hdr ctrl;
	virtio_net_ctrl_ack status = ~0;
	unsigned int tmp;
751
	int i;
752

753 754 755
	/* Caller should know better */
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
		(out + in > VIRTNET_SEND_COMMAND_SG_MAX));
756 757 758 759 760 761 762 763 764 765

	out++; /* Add header */
	in++; /* Add return status */

	ctrl.class = class;
	ctrl.cmd = cmd;

	sg_init_table(sg, out + in);

	sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
766 767
	for_each_sg(data, s, out + in - 2, i)
		sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
768 769
	sg_set_buf(&sg[out + in - 1], &status, sizeof(status));

770
	BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi, GFP_ATOMIC) < 0);
771

772
	virtqueue_kick(vi->cvq);
773 774 775 776 777

	/*
	 * Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
778
	while (!virtqueue_get_buf(vi->cvq, &tmp))
779 780 781 782 783
		cpu_relax();

	return status == VIRTIO_NET_OK;
}

R
Rusty Russell 已提交
784 785 786 787
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

788 789
	/* Make sure refill_work doesn't re-enable napi! */
	cancel_delayed_work_sync(&vi->refill);
R
Rusty Russell 已提交
790 791 792 793 794
	napi_disable(&vi->napi);

	return 0;
}

795 796 797
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
798
	struct scatterlist sg[2];
799
	u8 promisc, allmulti;
800
	struct virtio_net_ctrl_mac *mac_data;
J
Jiri Pirko 已提交
801
	struct netdev_hw_addr *ha;
802
	int uc_count;
803
	int mc_count;
804 805
	void *buf;
	int i;
806 807 808 809 810

	/* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

811 812
	promisc = ((dev->flags & IFF_PROMISC) != 0);
	allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
813

814
	sg_init_one(sg, &promisc, sizeof(promisc));
815 816 817

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_PROMISC,
818
				  sg, 1, 0))
819 820 821
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
			 promisc ? "en" : "dis");

822
	sg_init_one(sg, &allmulti, sizeof(allmulti));
823 824 825

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_ALLMULTI,
826
				  sg, 1, 0))
827 828
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
			 allmulti ? "en" : "dis");
829

830
	uc_count = netdev_uc_count(dev);
831
	mc_count = netdev_mc_count(dev);
832
	/* MAC filter - use one buffer for both lists */
833 834 835
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
836 837 838 839 840
	if (!buf) {
		dev_warn(&dev->dev, "No memory for MAC address buffer\n");
		return;
	}

841 842
	sg_init_table(sg, 2);

843
	/* Store the unicast list and count in the front of the buffer */
844
	mac_data->entries = uc_count;
J
Jiri Pirko 已提交
845
	i = 0;
846
	netdev_for_each_uc_addr(ha, dev)
J
Jiri Pirko 已提交
847
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
848 849

	sg_set_buf(&sg[0], mac_data,
850
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
851 852

	/* multicast list and count fill the end */
853
	mac_data = (void *)&mac_data->macs[uc_count][0];
854

855
	mac_data->entries = mc_count;
856
	i = 0;
857 858
	netdev_for_each_mc_addr(ha, dev)
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
859 860

	sg_set_buf(&sg[1], mac_data,
861
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
862 863 864 865 866 867 868

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
				  VIRTIO_NET_CTRL_MAC_TABLE_SET,
				  sg, 2, 0))
		dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");

	kfree(buf);
869 870
}

871
static int virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
872 873 874 875
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

876
	sg_init_one(&sg, &vid, sizeof(vid));
877 878 879 880

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
881
	return 0;
882 883
}

884
static int virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
885 886 887 888
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

889
	sg_init_one(&sg, &vid, sizeof(vid));
890 891 892 893

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
894
	return 0;
895 896
}

R
Rick Jones 已提交
897 898 899 900 901 902 903 904 905 906 907 908
static void virtnet_get_ringparam(struct net_device *dev,
				struct ethtool_ringparam *ring)
{
	struct virtnet_info *vi = netdev_priv(dev);

	ring->rx_max_pending = virtqueue_get_vring_size(vi->rvq);
	ring->tx_max_pending = virtqueue_get_vring_size(vi->svq);
	ring->rx_pending = ring->rx_max_pending;
	ring->tx_pending = ring->tx_max_pending;

}

909 910 911 912 913 914 915 916 917 918 919 920 921

static void virtnet_get_drvinfo(struct net_device *dev,
				struct ethtool_drvinfo *info)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;

	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
	strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
	strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));

}

922
static const struct ethtool_ops virtnet_ethtool_ops = {
923
	.get_drvinfo = virtnet_get_drvinfo,
924
	.get_link = ethtool_op_get_link,
R
Rick Jones 已提交
925
	.get_ringparam = virtnet_get_ringparam,
926 927
};

M
Mark McLoughlin 已提交
928 929 930 931 932 933 934 935 936 937 938
#define MIN_MTU 68
#define MAX_MTU 65535

static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
{
	if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
		return -EINVAL;
	dev->mtu = new_mtu;
	return 0;
}

939 940 941 942 943
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
944
	.ndo_set_mac_address = virtnet_set_mac_address,
945
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
946
	.ndo_change_mtu	     = virtnet_change_mtu,
947
	.ndo_get_stats64     = virtnet_stats,
948 949
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
950 951 952 953 954
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = virtnet_netpoll,
#endif
};

955 956 957 958
static void virtnet_update_status(struct virtnet_info *vi)
{
	u16 v;

959
	if (virtio_config_val(vi->vdev, VIRTIO_NET_F_STATUS,
960
			      offsetof(struct virtio_net_config, status),
961 962
			      &v) < 0)
		return;
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
		return;

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
		netif_carrier_on(vi->dev);
		netif_wake_queue(vi->dev);
	} else {
		netif_carrier_off(vi->dev);
		netif_stop_queue(vi->dev);
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	virtnet_update_status(vi);
}

988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
static int init_vqs(struct virtnet_info *vi)
{
	struct virtqueue *vqs[3];
	vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
	const char *names[] = { "input", "output", "control" };
	int nvqs, err;

	/* We expect two virtqueues, receive then send,
	 * and optionally control. */
	nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;

	err = vi->vdev->config->find_vqs(vi->vdev, nvqs, vqs, callbacks, names);
	if (err)
		return err;

	vi->rvq = vqs[0];
	vi->svq = vqs[1];

	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
		vi->cvq = vqs[2];

		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
			vi->dev->features |= NETIF_F_HW_VLAN_FILTER;
	}
	return 0;
}

R
Rusty Russell 已提交
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
static int virtnet_probe(struct virtio_device *vdev)
{
	int err;
	struct net_device *dev;
	struct virtnet_info *vi;

	/* Allocate ourselves a network device with room for our info */
	dev = alloc_etherdev(sizeof(struct virtnet_info));
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
1027
	dev->priv_flags |= IFF_UNICAST_FLT;
1028
	dev->netdev_ops = &virtnet_netdev;
R
Rusty Russell 已提交
1029
	dev->features = NETIF_F_HIGHDMA;
1030

1031
	SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
R
Rusty Russell 已提交
1032 1033 1034
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
1035
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
R
Rusty Russell 已提交
1036
		/* This opens up the world of extra features. */
1037 1038 1039 1040 1041 1042
		dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
		if (csum)
			dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;

		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
			dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
R
Rusty Russell 已提交
1043 1044
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
1045
		/* Individual feature bits: what can host handle? */
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
			dev->hw_features |= NETIF_F_TSO;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
			dev->hw_features |= NETIF_F_TSO6;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
			dev->hw_features |= NETIF_F_TSO_ECN;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
			dev->hw_features |= NETIF_F_UFO;

		if (gso)
			dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
		/* (!csum && gso) case will be fixed by register_netdev() */
R
Rusty Russell 已提交
1058 1059 1060
	}

	/* Configuration may specify what MAC to use.  Otherwise random. */
1061
	if (virtio_config_val_len(vdev, VIRTIO_NET_F_MAC,
1062
				  offsetof(struct virtio_net_config, mac),
1063
				  dev->dev_addr, dev->addr_len) < 0)
R
Rusty Russell 已提交
1064 1065 1066 1067
		random_ether_addr(dev->dev_addr);

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
1068
	netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
R
Rusty Russell 已提交
1069 1070
	vi->dev = dev;
	vi->vdev = vdev;
1071
	vdev->priv = vi;
1072
	vi->pages = NULL;
1073 1074 1075 1076 1077
	vi->stats = alloc_percpu(struct virtnet_stats);
	err = -ENOMEM;
	if (vi->stats == NULL)
		goto free;

1078
	INIT_DELAYED_WORK(&vi->refill, refill_work);
1079 1080
	sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
	sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
R
Rusty Russell 已提交
1081

1082
	/* If we can receive ANY GSO packets, we must allocate large ones. */
1083 1084 1085
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
1086 1087
		vi->big_packets = true;

1088 1089 1090
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

1091
	err = init_vqs(vi);
1092
	if (err)
1093
		goto free_stats;
R
Rusty Russell 已提交
1094 1095 1096 1097

	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
1098
		goto free_vqs;
R
Rusty Russell 已提交
1099
	}
1100 1101

	/* Last of all, set up some receive buffers. */
1102
	try_fill_recv(vi, GFP_KERNEL);
1103 1104 1105 1106 1107 1108 1109

	/* If we didn't even get one input buffer, we're useless. */
	if (vi->num == 0) {
		err = -ENOMEM;
		goto unregister;
	}

J
Jason Wang 已提交
1110 1111 1112 1113 1114 1115 1116 1117 1118
	/* Assume link up if device can't report link status,
	   otherwise get link status from config. */
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
		netif_carrier_off(dev);
		virtnet_update_status(vi);
	} else {
		vi->status = VIRTIO_NET_S_LINK_UP;
		netif_carrier_on(dev);
	}
1119

R
Rusty Russell 已提交
1120 1121 1122
	pr_debug("virtnet: registered device %s\n", dev->name);
	return 0;

1123 1124
unregister:
	unregister_netdev(dev);
1125 1126
free_vqs:
	vdev->config->del_vqs(vdev);
1127 1128
free_stats:
	free_percpu(vi->stats);
R
Rusty Russell 已提交
1129 1130 1131 1132 1133
free:
	free_netdev(dev);
	return err;
}

1134 1135 1136
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
S
Shirley Ma 已提交
1137
	while (1) {
1138
		buf = virtqueue_detach_unused_buf(vi->svq);
S
Shirley Ma 已提交
1139 1140 1141 1142
		if (!buf)
			break;
		dev_kfree_skb(buf);
	}
1143
	while (1) {
1144
		buf = virtqueue_detach_unused_buf(vi->rvq);
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
		if (!buf)
			break;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		--vi->num;
	}
	BUG_ON(vi->num != 0);
}

1156
static void remove_vq_common(struct virtnet_info *vi)
R
Rusty Russell 已提交
1157
{
1158
	vi->vdev->config->reset(vi->vdev);
S
Shirley Ma 已提交
1159 1160

	/* Free unused buffers in both send and recv, if any. */
1161
	free_unused_bufs(vi);
1162

1163
	vi->vdev->config->del_vqs(vi->vdev);
1164

1165 1166
	while (vi->pages)
		__free_pages(get_a_page(vi, GFP_KERNEL), 0);
1167 1168 1169 1170 1171 1172 1173 1174 1175
}

static void __devexit virtnet_remove(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	unregister_netdev(vi->dev);

	remove_vq_common(vi);
1176

1177
	free_percpu(vi->stats);
1178
	free_netdev(vi->dev);
R
Rusty Russell 已提交
1179 1180
}

1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
#ifdef CONFIG_PM
static int virtnet_freeze(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	virtqueue_disable_cb(vi->rvq);
	virtqueue_disable_cb(vi->svq);
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ))
		virtqueue_disable_cb(vi->cvq);

	netif_device_detach(vi->dev);
	cancel_delayed_work_sync(&vi->refill);

	if (netif_running(vi->dev))
		napi_disable(&vi->napi);

	remove_vq_common(vi);

	return 0;
}

static int virtnet_restore(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;
	int err;

	err = init_vqs(vi);
	if (err)
		return err;

	if (netif_running(vi->dev))
		virtnet_napi_enable(vi);

	netif_device_attach(vi->dev);

	if (!try_fill_recv(vi, GFP_KERNEL))
		queue_delayed_work(system_nrt_wq, &vi->refill, 0);

	return 0;
}
#endif

R
Rusty Russell 已提交
1223 1224 1225 1226 1227
static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

1228
static unsigned int features[] = {
1229 1230
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
	VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1231
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1232
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1233
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1234
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1235
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
1236 1237
};

1238
static struct virtio_driver virtio_net_driver = {
1239 1240
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
1241 1242 1243 1244 1245
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtnet_probe,
	.remove =	__devexit_p(virtnet_remove),
1246
	.config_changed = virtnet_config_changed,
1247 1248 1249 1250
#ifdef CONFIG_PM
	.freeze =	virtnet_freeze,
	.restore =	virtnet_restore,
#endif
R
Rusty Russell 已提交
1251 1252 1253 1254
};

static int __init init(void)
{
1255
	return register_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1256 1257 1258 1259
}

static void __exit fini(void)
{
1260
	unregister_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1261 1262 1263 1264 1265 1266 1267
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");