virtio_net.c 30.2 KB
Newer Older
1
/* A network driver using virtio.
R
Rusty Russell 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
22
#include <linux/ethtool.h>
R
Rusty Russell 已提交
23 24 25 26
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/scatterlist.h>
27
#include <linux/if_vlan.h>
28
#include <linux/slab.h>
R
Rusty Russell 已提交
29

30 31 32
static int napi_weight = 128;
module_param(napi_weight, int, 0444);

R
Rusty Russell 已提交
33 34 35 36
static int csum = 1, gso = 1;
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);

R
Rusty Russell 已提交
37
/* FIXME: MTU in config. */
38
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
39
#define GOOD_COPY_LEN	128
R
Rusty Russell 已提交
40

41
#define VIRTNET_SEND_COMMAND_SG_MAX    2
42
#define VIRTNET_DRIVER_VERSION "1.0.0"
43

44 45 46 47 48 49 50 51 52
struct virtnet_stats {
	struct u64_stats_sync syncp;
	u64 tx_bytes;
	u64 tx_packets;

	u64 rx_bytes;
	u64 rx_packets;
};

53
struct virtnet_info {
R
Rusty Russell 已提交
54
	struct virtio_device *vdev;
55
	struct virtqueue *rvq, *svq, *cvq;
R
Rusty Russell 已提交
56 57
	struct net_device *dev;
	struct napi_struct napi;
58
	unsigned int status;
R
Rusty Russell 已提交
59 60 61 62

	/* Number of input buffers, and max we've ever had. */
	unsigned int num, max;

63 64 65
	/* I like... big packets and I cannot lie! */
	bool big_packets;

66 67 68
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

69 70 71
	/* Active statistics */
	struct virtnet_stats __percpu *stats;

72 73 74
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

75 76
	/* Chain pages by the private ptr. */
	struct page *pages;
77 78 79 80

	/* fragments + linear part + virtio header */
	struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
	struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
R
Rusty Russell 已提交
81 82
};

83 84 85 86 87
struct skb_vnet_hdr {
	union {
		struct virtio_net_hdr hdr;
		struct virtio_net_hdr_mrg_rxbuf mhdr;
	};
88
	unsigned int num_sg;
89 90
};

91 92 93 94 95 96 97 98 99 100
struct padded_vnet_hdr {
	struct virtio_net_hdr hdr;
	/*
	 * virtio_net_hdr should be in a separated sg buffer because of a
	 * QEMU bug, and data sg buffer shares same page with this header sg.
	 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
	 */
	char padding[6];
};

101
static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
R
Rusty Russell 已提交
102
{
103
	return (struct skb_vnet_hdr *)skb->cb;
R
Rusty Russell 已提交
104 105
}

106 107 108 109 110
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
static void give_pages(struct virtnet_info *vi, struct page *page)
111
{
112
	struct page *end;
113

114 115 116 117
	/* Find end of list, sew whole thing into vi->pages. */
	for (end = page; end->private; end = (struct page *)end->private);
	end->private = (unsigned long)vi->pages;
	vi->pages = page;
118 119
}

120 121 122 123
static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
{
	struct page *p = vi->pages;

124
	if (p) {
125
		vi->pages = (struct page *)p->private;
126 127 128
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
129 130 131 132
		p = alloc_page(gfp_mask);
	return p;
}

133
static void skb_xmit_done(struct virtqueue *svq)
R
Rusty Russell 已提交
134
{
135
	struct virtnet_info *vi = svq->vdev->priv;
R
Rusty Russell 已提交
136

137
	/* Suppress further interrupts. */
138
	virtqueue_disable_cb(svq);
139

140
	/* We were probably waiting for more output buffers. */
R
Rusty Russell 已提交
141 142 143
	netif_wake_queue(vi->dev);
}

144 145
static void set_skb_frag(struct sk_buff *skb, struct page *page,
			 unsigned int offset, unsigned int *len)
R
Rusty Russell 已提交
146
{
147
	int size = min((unsigned)PAGE_SIZE - offset, *len);
148 149
	int i = skb_shinfo(skb)->nr_frags;

150
	__skb_fill_page_desc(skb, i, page, offset, size);
151

152 153
	skb->data_len += size;
	skb->len += size;
154
	skb->truesize += PAGE_SIZE;
155
	skb_shinfo(skb)->nr_frags++;
156
	*len -= size;
157
}
158

159 160 161 162 163 164 165
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct page *page, unsigned int len)
{
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
	unsigned int copy, hdr_len, offset;
	char *p;
166

167
	p = page_address(page);
168

169 170 171 172
	/* copy small packet so we can reuse these pages for small data */
	skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;
173

174
	hdr = skb_vnet_hdr(skb);
175

176 177 178 179 180 181 182
	if (vi->mergeable_rx_bufs) {
		hdr_len = sizeof hdr->mhdr;
		offset = hdr_len;
	} else {
		hdr_len = sizeof hdr->hdr;
		offset = sizeof(struct padded_vnet_hdr);
	}
183

184
	memcpy(hdr, p, hdr_len);
185

186 187
	len -= hdr_len;
	p += offset;
188

189 190 191 192
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	memcpy(skb_put(skb, copy), p, copy);
193

194 195
	len -= copy;
	offset += copy;
196

197 198 199 200 201 202 203 204 205 206 207 208 209
	/*
	 * Verify that we can indeed put this data into a skb.
	 * This is here to handle cases when the device erroneously
	 * tries to receive more than is possible. This is usually
	 * the case of a broken device.
	 */
	if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
		if (net_ratelimit())
			pr_debug("%s: too much data\n", skb->dev->name);
		dev_kfree_skb(skb);
		return NULL;
	}

210 211 212 213 214
	while (len) {
		set_skb_frag(skb, page, offset, &len);
		page = (struct page *)page->private;
		offset = 0;
	}
215

216 217
	if (page)
		give_pages(vi, page);
218

219 220
	return skb;
}
221

222 223 224 225 226 227 228 229 230 231 232 233 234 235
static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
{
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
	struct page *page;
	int num_buf, i, len;

	num_buf = hdr->mhdr.num_buffers;
	while (--num_buf) {
		i = skb_shinfo(skb)->nr_frags;
		if (i >= MAX_SKB_FRAGS) {
			pr_debug("%s: packet too long\n", skb->dev->name);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
		}
236
		page = virtqueue_get_buf(vi->rvq, &len);
237 238 239 240 241
		if (!page) {
			pr_debug("%s: rx error: %d buffers missing\n",
				 skb->dev->name, hdr->mhdr.num_buffers);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
242
		}
243

244 245 246 247 248 249 250 251 252 253 254 255 256
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;

		set_skb_frag(skb, page, 0, &len);

		--vi->num;
	}
	return 0;
}

static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
{
	struct virtnet_info *vi = netdev_priv(dev);
257
	struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
258 259 260
	struct sk_buff *skb;
	struct page *page;
	struct skb_vnet_hdr *hdr;
261

262 263 264 265 266 267 268 269 270
	if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		return;
	}
271

272 273 274 275 276 277 278 279
	if (!vi->mergeable_rx_bufs && !vi->big_packets) {
		skb = buf;
		len -= sizeof(struct virtio_net_hdr);
		skb_trim(skb, len);
	} else {
		page = buf;
		skb = page_to_skb(vi, page, len);
		if (unlikely(!skb)) {
280
			dev->stats.rx_dropped++;
281 282
			give_pages(vi, page);
			return;
283
		}
284 285 286 287 288
		if (vi->mergeable_rx_bufs)
			if (receive_mergeable(vi, skb)) {
				dev_kfree_skb(skb);
				return;
			}
289
	}
290

291
	hdr = skb_vnet_hdr(skb);
292 293 294 295 296

	u64_stats_update_begin(&stats->syncp);
	stats->rx_bytes += skb->len;
	stats->rx_packets++;
	u64_stats_update_end(&stats->syncp);
R
Rusty Russell 已提交
297

298
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
R
Rusty Russell 已提交
299
		pr_debug("Needs csum!\n");
300 301 302
		if (!skb_partial_csum_set(skb,
					  hdr->hdr.csum_start,
					  hdr->hdr.csum_offset))
R
Rusty Russell 已提交
303
			goto frame_err;
304 305
	} else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
		skb->ip_summed = CHECKSUM_UNNECESSARY;
R
Rusty Russell 已提交
306 307
	}

308 309 310 311
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

312
	if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
R
Rusty Russell 已提交
313
		pr_debug("GSO!\n");
314
		switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
R
Rusty Russell 已提交
315 316 317 318 319 320 321 322 323 324 325 326
		case VIRTIO_NET_HDR_GSO_TCPV4:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
			break;
		case VIRTIO_NET_HDR_GSO_UDP:
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
			break;
		case VIRTIO_NET_HDR_GSO_TCPV6:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
			break;
		default:
			if (net_ratelimit())
				printk(KERN_WARNING "%s: bad gso type %u.\n",
327
				       dev->name, hdr->hdr.gso_type);
R
Rusty Russell 已提交
328 329 330
			goto frame_err;
		}

331
		if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
R
Rusty Russell 已提交
332 333
			skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;

334
		skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
R
Rusty Russell 已提交
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
		if (skb_shinfo(skb)->gso_size == 0) {
			if (net_ratelimit())
				printk(KERN_WARNING "%s: zero gso size.\n",
				       dev->name);
			goto frame_err;
		}

		/* Header must be checked, and gso_segs computed. */
		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
		skb_shinfo(skb)->gso_segs = 0;
	}

	netif_receive_skb(skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

355
static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
R
Rusty Russell 已提交
356 357
{
	struct sk_buff *skb;
358 359
	struct skb_vnet_hdr *hdr;
	int err;
360

361 362 363
	skb = netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN);
	if (unlikely(!skb))
		return -ENOMEM;
R
Rusty Russell 已提交
364

365
	skb_put(skb, MAX_PACKET_LEN);
366

367
	hdr = skb_vnet_hdr(skb);
368
	sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
369

370
	skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
371

372
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 2, skb, gfp);
373 374
	if (err < 0)
		dev_kfree_skb(skb);
375

376 377
	return err;
}
378

379 380 381 382 383 384
static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

385
	/* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
386 387 388 389 390 391
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
		first = get_a_page(vi, gfp);
		if (!first) {
			if (list)
				give_pages(vi, list);
			return -ENOMEM;
392
		}
393
		sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
394

395 396 397 398
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
R
Rusty Russell 已提交
399

400 401 402 403 404 405 406
	first = get_a_page(vi, gfp);
	if (!first) {
		give_pages(vi, list);
		return -ENOMEM;
	}
	p = page_address(first);

407 408 409
	/* vi->rx_sg[0], vi->rx_sg[1] share the same page */
	/* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
	sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
410

411
	/* vi->rx_sg[1] for data packet, from offset */
412
	offset = sizeof(struct padded_vnet_hdr);
413
	sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
414 415 416

	/* chain first in list head */
	first->private = (unsigned long)list;
417 418
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
				    first, gfp);
419 420 421 422
	if (err < 0)
		give_pages(vi, first);

	return err;
R
Rusty Russell 已提交
423 424
}

425
static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
426
{
427
	struct page *page;
428 429
	int err;

430 431 432
	page = get_a_page(vi, gfp);
	if (!page)
		return -ENOMEM;
433

434
	sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
435

436
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 1, page, gfp);
437 438
	if (err < 0)
		give_pages(vi, page);
439

440 441
	return err;
}
442

443 444 445 446
/* Returns false if we couldn't fill entirely (OOM). */
static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
{
	int err;
447
	bool oom;
448

449 450 451 452 453 454 455
	do {
		if (vi->mergeable_rx_bufs)
			err = add_recvbuf_mergeable(vi, gfp);
		else if (vi->big_packets)
			err = add_recvbuf_big(vi, gfp);
		else
			err = add_recvbuf_small(vi, gfp);
456

457 458
		oom = err == -ENOMEM;
		if (err < 0)
459
			break;
460
		++vi->num;
461
	} while (err > 0);
462 463
	if (unlikely(vi->num > vi->max))
		vi->max = vi->num;
464
	virtqueue_kick(vi->rvq);
465
	return !oom;
466 467
}

468
static void skb_recv_done(struct virtqueue *rvq)
R
Rusty Russell 已提交
469 470
{
	struct virtnet_info *vi = rvq->vdev->priv;
471
	/* Schedule NAPI, Suppress further interrupts if successful. */
472
	if (napi_schedule_prep(&vi->napi)) {
473
		virtqueue_disable_cb(rvq);
474
		__napi_schedule(&vi->napi);
475
	}
R
Rusty Russell 已提交
476 477
}

478 479 480 481 482 483 484 485 486 487 488 489 490 491
static void virtnet_napi_enable(struct virtnet_info *vi)
{
	napi_enable(&vi->napi);

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets
	 * now.  virtnet_poll wants re-enable the queue, so we disable here.
	 * We synchronize against interrupts via NAPI_STATE_SCHED */
	if (napi_schedule_prep(&vi->napi)) {
		virtqueue_disable_cb(vi->rvq);
		__napi_schedule(&vi->napi);
	}
}

492 493 494 495 496 497 498
static void refill_work(struct work_struct *work)
{
	struct virtnet_info *vi;
	bool still_empty;

	vi = container_of(work, struct virtnet_info, refill.work);
	napi_disable(&vi->napi);
499
	still_empty = !try_fill_recv(vi, GFP_KERNEL);
500
	virtnet_napi_enable(vi);
501 502 503 504 505 506 507

	/* In theory, this can happen: if we don't get any buffers in
	 * we will *never* try to fill again. */
	if (still_empty)
		schedule_delayed_work(&vi->refill, HZ/2);
}

R
Rusty Russell 已提交
508 509 510
static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
511
	void *buf;
R
Rusty Russell 已提交
512 513 514 515
	unsigned int len, received = 0;

again:
	while (received < budget &&
516
	       (buf = virtqueue_get_buf(vi->rvq, &len)) != NULL) {
517 518
		receive_buf(vi->dev, buf, len);
		--vi->num;
R
Rusty Russell 已提交
519 520 521
		received++;
	}

522 523 524 525
	if (vi->num < vi->max / 2) {
		if (!try_fill_recv(vi, GFP_ATOMIC))
			schedule_delayed_work(&vi->refill, 0);
	}
R
Rusty Russell 已提交
526

527 528
	/* Out of packets? */
	if (received < budget) {
529
		napi_complete(napi);
530
		if (unlikely(!virtqueue_enable_cb(vi->rvq)) &&
531
		    napi_schedule_prep(napi)) {
532
			virtqueue_disable_cb(vi->rvq);
533
			__napi_schedule(napi);
R
Rusty Russell 已提交
534
			goto again;
535
		}
R
Rusty Russell 已提交
536 537 538 539 540
	}

	return received;
}

541
static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
R
Rusty Russell 已提交
542 543
{
	struct sk_buff *skb;
544
	unsigned int len, tot_sgs = 0;
545
	struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
R
Rusty Russell 已提交
546

547
	while ((skb = virtqueue_get_buf(vi->svq, &len)) != NULL) {
R
Rusty Russell 已提交
548
		pr_debug("Sent skb %p\n", skb);
549 550 551 552 553 554

		u64_stats_update_begin(&stats->syncp);
		stats->tx_bytes += skb->len;
		stats->tx_packets++;
		u64_stats_update_end(&stats->syncp);

555
		tot_sgs += skb_vnet_hdr(skb)->num_sg;
556
		dev_kfree_skb_any(skb);
R
Rusty Russell 已提交
557
	}
558
	return tot_sgs;
R
Rusty Russell 已提交
559 560
}

561
static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
R
Rusty Russell 已提交
562
{
563
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
R
Rusty Russell 已提交
564 565
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;

J
Johannes Berg 已提交
566
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
R
Rusty Russell 已提交
567 568

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
569
		hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
570
		hdr->hdr.csum_start = skb_checksum_start_offset(skb);
571
		hdr->hdr.csum_offset = skb->csum_offset;
R
Rusty Russell 已提交
572
	} else {
573 574
		hdr->hdr.flags = 0;
		hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
R
Rusty Russell 已提交
575 576 577
	}

	if (skb_is_gso(skb)) {
578 579
		hdr->hdr.hdr_len = skb_headlen(skb);
		hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
R
Rusty Russell 已提交
580
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
581
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
R
Rusty Russell 已提交
582
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
583
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
R
Rusty Russell 已提交
584
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
585
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
R
Rusty Russell 已提交
586 587
		else
			BUG();
R
Rusty Russell 已提交
588
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
589
			hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
R
Rusty Russell 已提交
590
	} else {
591 592
		hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
		hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
R
Rusty Russell 已提交
593 594
	}

595
	hdr->mhdr.num_buffers = 0;
596 597 598

	/* Encode metadata header at front. */
	if (vi->mergeable_rx_bufs)
599
		sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
600
	else
601
		sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
602

603
	hdr->num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
604
	return virtqueue_add_buf(vi->svq, vi->tx_sg, hdr->num_sg,
605
					0, skb);
606 607
}

608
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
609 610
{
	struct virtnet_info *vi = netdev_priv(dev);
611
	int capacity;
612 613 614

	/* Free up any pending old buffers before queueing new ones. */
	free_old_xmit_skbs(vi);
615

616
	/* Try to transmit */
617 618 619 620
	capacity = xmit_skb(vi, skb);

	/* This can happen with OOM and indirect buffers. */
	if (unlikely(capacity < 0)) {
621 622 623 624 625 626 627 628 629 630
		if (net_ratelimit()) {
			if (likely(capacity == -ENOMEM)) {
				dev_warn(&dev->dev,
					 "TX queue failure: out of memory\n");
			} else {
				dev->stats.tx_fifo_errors++;
				dev_warn(&dev->dev,
					 "Unexpected TX queue failure: %d\n",
					 capacity);
			}
631
		}
632 633 634
		dev->stats.tx_dropped++;
		kfree_skb(skb);
		return NETDEV_TX_OK;
R
Rusty Russell 已提交
635
	}
636
	virtqueue_kick(vi->svq);
637

638 639 640 641 642 643 644 645
	/* Don't wait up for transmitted skbs to be freed. */
	skb_orphan(skb);
	nf_reset(skb);

	/* Apparently nice girls don't return TX_BUSY; stop the queue
	 * before it gets out of hand.  Naturally, this wastes entries. */
	if (capacity < 2+MAX_SKB_FRAGS) {
		netif_stop_queue(dev);
646
		if (unlikely(!virtqueue_enable_cb_delayed(vi->svq))) {
647 648 649 650
			/* More just got used, free them then recheck. */
			capacity += free_old_xmit_skbs(vi);
			if (capacity >= 2+MAX_SKB_FRAGS) {
				netif_start_queue(dev);
651
				virtqueue_disable_cb(vi->svq);
652 653
			}
		}
654
	}
655 656

	return NETDEV_TX_OK;
R
Rusty Russell 已提交
657 658
}

659 660 661 662 663 664 665 666 667 668
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
	int ret;

	ret = eth_mac_addr(dev, p);
	if (ret)
		return ret;

669 670 671
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
		                  dev->dev_addr, dev->addr_len);
672 673 674 675

	return 0;
}

676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
					       struct rtnl_link_stats64 *tot)
{
	struct virtnet_info *vi = netdev_priv(dev);
	int cpu;
	unsigned int start;

	for_each_possible_cpu(cpu) {
		struct virtnet_stats __percpu *stats
			= per_cpu_ptr(vi->stats, cpu);
		u64 tpackets, tbytes, rpackets, rbytes;

		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			tpackets = stats->tx_packets;
			tbytes   = stats->tx_bytes;
			rpackets = stats->rx_packets;
			rbytes   = stats->rx_bytes;
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		tot->rx_packets += rpackets;
		tot->tx_packets += tpackets;
		tot->rx_bytes   += rbytes;
		tot->tx_bytes   += tbytes;
	}

	tot->tx_dropped = dev->stats.tx_dropped;
703
	tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
704 705 706 707 708 709 710
	tot->rx_dropped = dev->stats.rx_dropped;
	tot->rx_length_errors = dev->stats.rx_length_errors;
	tot->rx_frame_errors = dev->stats.rx_frame_errors;

	return tot;
}

711 712 713 714 715 716 717 718 719
#ifdef CONFIG_NET_POLL_CONTROLLER
static void virtnet_netpoll(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_schedule(&vi->napi);
}
#endif

R
Rusty Russell 已提交
720 721 722 723
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

724
	virtnet_napi_enable(vi);
R
Rusty Russell 已提交
725 726 727
	return 0;
}

728 729 730 731 732 733 734 735
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
 * never fail unless improperly formated.
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
				 struct scatterlist *data, int out, int in)
{
736
	struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
737 738 739
	struct virtio_net_ctrl_hdr ctrl;
	virtio_net_ctrl_ack status = ~0;
	unsigned int tmp;
740
	int i;
741

742 743 744
	/* Caller should know better */
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
		(out + in > VIRTNET_SEND_COMMAND_SG_MAX));
745 746 747 748 749 750 751 752 753 754

	out++; /* Add header */
	in++; /* Add return status */

	ctrl.class = class;
	ctrl.cmd = cmd;

	sg_init_table(sg, out + in);

	sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
755 756
	for_each_sg(data, s, out + in - 2, i)
		sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
757 758
	sg_set_buf(&sg[out + in - 1], &status, sizeof(status));

759
	BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi) < 0);
760

761
	virtqueue_kick(vi->cvq);
762 763 764 765 766

	/*
	 * Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
767
	while (!virtqueue_get_buf(vi->cvq, &tmp))
768 769 770 771 772
		cpu_relax();

	return status == VIRTIO_NET_OK;
}

R
Rusty Russell 已提交
773 774 775 776 777 778 779 780 781
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_disable(&vi->napi);

	return 0;
}

782 783 784
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
785
	struct scatterlist sg[2];
786
	u8 promisc, allmulti;
787
	struct virtio_net_ctrl_mac *mac_data;
J
Jiri Pirko 已提交
788
	struct netdev_hw_addr *ha;
789
	int uc_count;
790
	int mc_count;
791 792
	void *buf;
	int i;
793 794 795 796 797

	/* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

798 799
	promisc = ((dev->flags & IFF_PROMISC) != 0);
	allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
800

801
	sg_init_one(sg, &promisc, sizeof(promisc));
802 803 804

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_PROMISC,
805
				  sg, 1, 0))
806 807 808
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
			 promisc ? "en" : "dis");

809
	sg_init_one(sg, &allmulti, sizeof(allmulti));
810 811 812

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_ALLMULTI,
813
				  sg, 1, 0))
814 815
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
			 allmulti ? "en" : "dis");
816

817
	uc_count = netdev_uc_count(dev);
818
	mc_count = netdev_mc_count(dev);
819
	/* MAC filter - use one buffer for both lists */
820 821 822
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
823 824 825 826 827
	if (!buf) {
		dev_warn(&dev->dev, "No memory for MAC address buffer\n");
		return;
	}

828 829
	sg_init_table(sg, 2);

830
	/* Store the unicast list and count in the front of the buffer */
831
	mac_data->entries = uc_count;
J
Jiri Pirko 已提交
832
	i = 0;
833
	netdev_for_each_uc_addr(ha, dev)
J
Jiri Pirko 已提交
834
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
835 836

	sg_set_buf(&sg[0], mac_data,
837
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
838 839

	/* multicast list and count fill the end */
840
	mac_data = (void *)&mac_data->macs[uc_count][0];
841

842
	mac_data->entries = mc_count;
843
	i = 0;
844 845
	netdev_for_each_mc_addr(ha, dev)
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
846 847

	sg_set_buf(&sg[1], mac_data,
848
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
849 850 851 852 853 854 855

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
				  VIRTIO_NET_CTRL_MAC_TABLE_SET,
				  sg, 2, 0))
		dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");

	kfree(buf);
856 857
}

858
static void virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
859 860 861 862
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

863
	sg_init_one(&sg, &vid, sizeof(vid));
864 865 866 867 868 869

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
}

870
static void virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
871 872 873 874
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

875
	sg_init_one(&sg, &vid, sizeof(vid));
876 877 878 879 880 881

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
}

R
Rick Jones 已提交
882 883 884 885 886 887 888 889 890 891 892 893
static void virtnet_get_ringparam(struct net_device *dev,
				struct ethtool_ringparam *ring)
{
	struct virtnet_info *vi = netdev_priv(dev);

	ring->rx_max_pending = virtqueue_get_vring_size(vi->rvq);
	ring->tx_max_pending = virtqueue_get_vring_size(vi->svq);
	ring->rx_pending = ring->rx_max_pending;
	ring->tx_pending = ring->tx_max_pending;

}

894 895 896 897 898 899 900 901 902 903 904 905 906

static void virtnet_get_drvinfo(struct net_device *dev,
				struct ethtool_drvinfo *info)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;

	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
	strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
	strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));

}

907
static const struct ethtool_ops virtnet_ethtool_ops = {
908
	.get_drvinfo = virtnet_get_drvinfo,
909
	.get_link = ethtool_op_get_link,
R
Rick Jones 已提交
910
	.get_ringparam = virtnet_get_ringparam,
911 912
};

M
Mark McLoughlin 已提交
913 914 915 916 917 918 919 920 921 922 923
#define MIN_MTU 68
#define MAX_MTU 65535

static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
{
	if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
		return -EINVAL;
	dev->mtu = new_mtu;
	return 0;
}

924 925 926 927 928
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
929
	.ndo_set_mac_address = virtnet_set_mac_address,
930
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
931
	.ndo_change_mtu	     = virtnet_change_mtu,
932
	.ndo_get_stats64     = virtnet_stats,
933 934
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
935 936 937 938 939
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = virtnet_netpoll,
#endif
};

940 941 942 943
static void virtnet_update_status(struct virtnet_info *vi)
{
	u16 v;

944
	if (virtio_config_val(vi->vdev, VIRTIO_NET_F_STATUS,
945
			      offsetof(struct virtio_net_config, status),
946 947
			      &v) < 0)
		return;
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
		return;

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
		netif_carrier_on(vi->dev);
		netif_wake_queue(vi->dev);
	} else {
		netif_carrier_off(vi->dev);
		netif_stop_queue(vi->dev);
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	virtnet_update_status(vi);
}

R
Rusty Russell 已提交
973 974 975 976 977
static int virtnet_probe(struct virtio_device *vdev)
{
	int err;
	struct net_device *dev;
	struct virtnet_info *vi;
978 979 980 981
	struct virtqueue *vqs[3];
	vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
	const char *names[] = { "input", "output", "control" };
	int nvqs;
R
Rusty Russell 已提交
982 983 984 985 986 987 988

	/* Allocate ourselves a network device with room for our info */
	dev = alloc_etherdev(sizeof(struct virtnet_info));
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
989
	dev->priv_flags |= IFF_UNICAST_FLT;
990
	dev->netdev_ops = &virtnet_netdev;
R
Rusty Russell 已提交
991
	dev->features = NETIF_F_HIGHDMA;
992

993
	SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
R
Rusty Russell 已提交
994 995 996
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
997
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
R
Rusty Russell 已提交
998
		/* This opens up the world of extra features. */
999 1000 1001 1002 1003 1004
		dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
		if (csum)
			dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;

		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
			dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
R
Rusty Russell 已提交
1005 1006
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
1007
		/* Individual feature bits: what can host handle? */
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
			dev->hw_features |= NETIF_F_TSO;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
			dev->hw_features |= NETIF_F_TSO6;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
			dev->hw_features |= NETIF_F_TSO_ECN;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
			dev->hw_features |= NETIF_F_UFO;

		if (gso)
			dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
		/* (!csum && gso) case will be fixed by register_netdev() */
R
Rusty Russell 已提交
1020 1021 1022
	}

	/* Configuration may specify what MAC to use.  Otherwise random. */
1023
	if (virtio_config_val_len(vdev, VIRTIO_NET_F_MAC,
1024
				  offsetof(struct virtio_net_config, mac),
1025
				  dev->dev_addr, dev->addr_len) < 0)
R
Rusty Russell 已提交
1026 1027 1028 1029
		random_ether_addr(dev->dev_addr);

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
1030
	netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
R
Rusty Russell 已提交
1031 1032
	vi->dev = dev;
	vi->vdev = vdev;
1033
	vdev->priv = vi;
1034
	vi->pages = NULL;
1035 1036 1037 1038 1039
	vi->stats = alloc_percpu(struct virtnet_stats);
	err = -ENOMEM;
	if (vi->stats == NULL)
		goto free;

1040
	INIT_DELAYED_WORK(&vi->refill, refill_work);
1041 1042
	sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
	sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
R
Rusty Russell 已提交
1043

1044
	/* If we can receive ANY GSO packets, we must allocate large ones. */
1045 1046 1047
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
1048 1049
		vi->big_packets = true;

1050 1051 1052
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

1053 1054 1055 1056 1057 1058
	/* We expect two virtqueues, receive then send,
	 * and optionally control. */
	nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;

	err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
	if (err)
1059
		goto free_stats;
R
Rusty Russell 已提交
1060

1061 1062
	vi->rvq = vqs[0];
	vi->svq = vqs[1];
R
Rusty Russell 已提交
1063

1064
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
1065
		vi->cvq = vqs[2];
1066 1067 1068

		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
			dev->features |= NETIF_F_HW_VLAN_FILTER;
1069 1070
	}

R
Rusty Russell 已提交
1071 1072 1073
	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
1074
		goto free_vqs;
R
Rusty Russell 已提交
1075
	}
1076 1077

	/* Last of all, set up some receive buffers. */
1078
	try_fill_recv(vi, GFP_KERNEL);
1079 1080 1081 1082 1083 1084 1085

	/* If we didn't even get one input buffer, we're useless. */
	if (vi->num == 0) {
		err = -ENOMEM;
		goto unregister;
	}

J
Jason Wang 已提交
1086 1087 1088 1089 1090 1091 1092 1093 1094
	/* Assume link up if device can't report link status,
	   otherwise get link status from config. */
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
		netif_carrier_off(dev);
		virtnet_update_status(vi);
	} else {
		vi->status = VIRTIO_NET_S_LINK_UP;
		netif_carrier_on(dev);
	}
1095

R
Rusty Russell 已提交
1096 1097 1098
	pr_debug("virtnet: registered device %s\n", dev->name);
	return 0;

1099 1100
unregister:
	unregister_netdev(dev);
1101
	cancel_delayed_work_sync(&vi->refill);
1102 1103
free_vqs:
	vdev->config->del_vqs(vdev);
1104 1105
free_stats:
	free_percpu(vi->stats);
R
Rusty Russell 已提交
1106 1107 1108 1109 1110
free:
	free_netdev(dev);
	return err;
}

1111 1112 1113
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
S
Shirley Ma 已提交
1114
	while (1) {
1115
		buf = virtqueue_detach_unused_buf(vi->svq);
S
Shirley Ma 已提交
1116 1117 1118 1119
		if (!buf)
			break;
		dev_kfree_skb(buf);
	}
1120
	while (1) {
1121
		buf = virtqueue_detach_unused_buf(vi->rvq);
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
		if (!buf)
			break;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		--vi->num;
	}
	BUG_ON(vi->num != 0);
}

1133
static void __devexit virtnet_remove(struct virtio_device *vdev)
R
Rusty Russell 已提交
1134
{
1135
	struct virtnet_info *vi = vdev->priv;
1136

R
Rusty Russell 已提交
1137 1138 1139
	/* Stop all the virtqueues. */
	vdev->config->reset(vdev);

1140

1141
	unregister_netdev(vi->dev);
1142
	cancel_delayed_work_sync(&vi->refill);
S
Shirley Ma 已提交
1143 1144

	/* Free unused buffers in both send and recv, if any. */
1145
	free_unused_bufs(vi);
1146

1147 1148
	vdev->config->del_vqs(vi->vdev);

1149 1150 1151
	while (vi->pages)
		__free_pages(get_a_page(vi, GFP_KERNEL), 0);

1152
	free_percpu(vi->stats);
1153
	free_netdev(vi->dev);
R
Rusty Russell 已提交
1154 1155 1156 1157 1158 1159 1160
}

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

1161
static unsigned int features[] = {
1162 1163
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
	VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1164
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1165
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1166
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1167
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1168
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
1169 1170
};

1171
static struct virtio_driver virtio_net_driver = {
1172 1173
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
1174 1175 1176 1177 1178
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtnet_probe,
	.remove =	__devexit_p(virtnet_remove),
1179
	.config_changed = virtnet_config_changed,
R
Rusty Russell 已提交
1180 1181 1182 1183
};

static int __init init(void)
{
1184
	return register_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1185 1186 1187 1188
}

static void __exit fini(void)
{
1189
	unregister_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1190 1191 1192 1193 1194 1195 1196
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");