virtio_net.c 27.3 KB
Newer Older
1
/* A network driver using virtio.
R
Rusty Russell 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
22
#include <linux/ethtool.h>
R
Rusty Russell 已提交
23 24 25 26
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/scatterlist.h>
27
#include <linux/if_vlan.h>
28
#include <linux/slab.h>
R
Rusty Russell 已提交
29

30 31 32
static int napi_weight = 128;
module_param(napi_weight, int, 0444);

R
Rusty Russell 已提交
33 34 35 36
static int csum = 1, gso = 1;
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);

R
Rusty Russell 已提交
37
/* FIXME: MTU in config. */
38
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
39
#define GOOD_COPY_LEN	128
R
Rusty Russell 已提交
40

41
#define VIRTNET_SEND_COMMAND_SG_MAX    2
42

43
struct virtnet_info {
R
Rusty Russell 已提交
44
	struct virtio_device *vdev;
45
	struct virtqueue *rvq, *svq, *cvq;
R
Rusty Russell 已提交
46 47
	struct net_device *dev;
	struct napi_struct napi;
48
	unsigned int status;
R
Rusty Russell 已提交
49 50 51 52

	/* Number of input buffers, and max we've ever had. */
	unsigned int num, max;

53 54 55
	/* I like... big packets and I cannot lie! */
	bool big_packets;

56 57 58
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

59 60 61
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

62 63
	/* Chain pages by the private ptr. */
	struct page *pages;
64 65 66 67

	/* fragments + linear part + virtio header */
	struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
	struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
R
Rusty Russell 已提交
68 69
};

70 71 72 73 74
struct skb_vnet_hdr {
	union {
		struct virtio_net_hdr hdr;
		struct virtio_net_hdr_mrg_rxbuf mhdr;
	};
75
	unsigned int num_sg;
76 77
};

78 79 80 81 82 83 84 85 86 87
struct padded_vnet_hdr {
	struct virtio_net_hdr hdr;
	/*
	 * virtio_net_hdr should be in a separated sg buffer because of a
	 * QEMU bug, and data sg buffer shares same page with this header sg.
	 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
	 */
	char padding[6];
};

88
static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
R
Rusty Russell 已提交
89
{
90
	return (struct skb_vnet_hdr *)skb->cb;
R
Rusty Russell 已提交
91 92
}

93 94 95 96 97
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
static void give_pages(struct virtnet_info *vi, struct page *page)
98
{
99
	struct page *end;
100

101 102 103 104
	/* Find end of list, sew whole thing into vi->pages. */
	for (end = page; end->private; end = (struct page *)end->private);
	end->private = (unsigned long)vi->pages;
	vi->pages = page;
105 106
}

107 108 109 110
static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
{
	struct page *p = vi->pages;

111
	if (p) {
112
		vi->pages = (struct page *)p->private;
113 114 115
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
116 117 118 119
		p = alloc_page(gfp_mask);
	return p;
}

120
static void skb_xmit_done(struct virtqueue *svq)
R
Rusty Russell 已提交
121
{
122
	struct virtnet_info *vi = svq->vdev->priv;
R
Rusty Russell 已提交
123

124 125
	/* Suppress further interrupts. */
	svq->vq_ops->disable_cb(svq);
126

127
	/* We were probably waiting for more output buffers. */
R
Rusty Russell 已提交
128 129 130
	netif_wake_queue(vi->dev);
}

131 132
static void set_skb_frag(struct sk_buff *skb, struct page *page,
			 unsigned int offset, unsigned int *len)
R
Rusty Russell 已提交
133
{
134 135 136 137 138 139 140 141 142 143 144 145 146
	int i = skb_shinfo(skb)->nr_frags;
	skb_frag_t *f;

	f = &skb_shinfo(skb)->frags[i];
	f->size = min((unsigned)PAGE_SIZE - offset, *len);
	f->page_offset = offset;
	f->page = page;

	skb->data_len += f->size;
	skb->len += f->size;
	skb_shinfo(skb)->nr_frags++;
	*len -= f->size;
}
147

148 149 150 151 152 153 154
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct page *page, unsigned int len)
{
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
	unsigned int copy, hdr_len, offset;
	char *p;
155

156
	p = page_address(page);
157

158 159 160 161
	/* copy small packet so we can reuse these pages for small data */
	skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;
162

163
	hdr = skb_vnet_hdr(skb);
164

165 166 167 168 169 170 171
	if (vi->mergeable_rx_bufs) {
		hdr_len = sizeof hdr->mhdr;
		offset = hdr_len;
	} else {
		hdr_len = sizeof hdr->hdr;
		offset = sizeof(struct padded_vnet_hdr);
	}
172

173
	memcpy(hdr, p, hdr_len);
174

175 176
	len -= hdr_len;
	p += offset;
177

178 179 180 181
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	memcpy(skb_put(skb, copy), p, copy);
182

183 184
	len -= copy;
	offset += copy;
185

186 187 188 189 190
	while (len) {
		set_skb_frag(skb, page, offset, &len);
		page = (struct page *)page->private;
		offset = 0;
	}
191

192 193
	if (page)
		give_pages(vi, page);
194

195 196
	return skb;
}
197

198 199 200 201 202 203 204 205 206 207 208 209 210 211
static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
{
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
	struct page *page;
	int num_buf, i, len;

	num_buf = hdr->mhdr.num_buffers;
	while (--num_buf) {
		i = skb_shinfo(skb)->nr_frags;
		if (i >= MAX_SKB_FRAGS) {
			pr_debug("%s: packet too long\n", skb->dev->name);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
		}
212

213 214 215 216 217 218
		page = vi->rvq->vq_ops->get_buf(vi->rvq, &len);
		if (!page) {
			pr_debug("%s: rx error: %d buffers missing\n",
				 skb->dev->name, hdr->mhdr.num_buffers);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
219
		}
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;

		set_skb_frag(skb, page, 0, &len);

		--vi->num;
	}
	return 0;
}

static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct sk_buff *skb;
	struct page *page;
	struct skb_vnet_hdr *hdr;
236

237 238 239 240 241 242 243 244 245
	if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		return;
	}
246

247 248 249 250 251 252 253 254
	if (!vi->mergeable_rx_bufs && !vi->big_packets) {
		skb = buf;
		len -= sizeof(struct virtio_net_hdr);
		skb_trim(skb, len);
	} else {
		page = buf;
		skb = page_to_skb(vi, page, len);
		if (unlikely(!skb)) {
255
			dev->stats.rx_dropped++;
256 257
			give_pages(vi, page);
			return;
258
		}
259 260 261 262 263
		if (vi->mergeable_rx_bufs)
			if (receive_mergeable(vi, skb)) {
				dev_kfree_skb(skb);
				return;
			}
264
	}
265

266
	hdr = skb_vnet_hdr(skb);
267
	skb->truesize += skb->data_len;
R
Rusty Russell 已提交
268 269 270
	dev->stats.rx_bytes += skb->len;
	dev->stats.rx_packets++;

271
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
R
Rusty Russell 已提交
272
		pr_debug("Needs csum!\n");
273 274 275
		if (!skb_partial_csum_set(skb,
					  hdr->hdr.csum_start,
					  hdr->hdr.csum_offset))
R
Rusty Russell 已提交
276 277 278
			goto frame_err;
	}

279 280 281 282
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

283
	if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
R
Rusty Russell 已提交
284
		pr_debug("GSO!\n");
285
		switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
R
Rusty Russell 已提交
286 287 288 289 290 291 292 293 294 295 296 297
		case VIRTIO_NET_HDR_GSO_TCPV4:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
			break;
		case VIRTIO_NET_HDR_GSO_UDP:
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
			break;
		case VIRTIO_NET_HDR_GSO_TCPV6:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
			break;
		default:
			if (net_ratelimit())
				printk(KERN_WARNING "%s: bad gso type %u.\n",
298
				       dev->name, hdr->hdr.gso_type);
R
Rusty Russell 已提交
299 300 301
			goto frame_err;
		}

302
		if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
R
Rusty Russell 已提交
303 304
			skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;

305
		skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
R
Rusty Russell 已提交
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
		if (skb_shinfo(skb)->gso_size == 0) {
			if (net_ratelimit())
				printk(KERN_WARNING "%s: zero gso size.\n",
				       dev->name);
			goto frame_err;
		}

		/* Header must be checked, and gso_segs computed. */
		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
		skb_shinfo(skb)->gso_segs = 0;
	}

	netif_receive_skb(skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

326
static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
R
Rusty Russell 已提交
327 328
{
	struct sk_buff *skb;
329 330
	struct skb_vnet_hdr *hdr;
	int err;
331

332 333 334
	skb = netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN);
	if (unlikely(!skb))
		return -ENOMEM;
R
Rusty Russell 已提交
335

336
	skb_put(skb, MAX_PACKET_LEN);
337

338
	hdr = skb_vnet_hdr(skb);
339
	sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
340

341
	skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
342

343
	err = vi->rvq->vq_ops->add_buf(vi->rvq, vi->rx_sg, 0, 2, skb);
344 345
	if (err < 0)
		dev_kfree_skb(skb);
346

347 348
	return err;
}
349

350 351 352 353 354 355
static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

356
	/* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
357 358 359 360 361 362
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
		first = get_a_page(vi, gfp);
		if (!first) {
			if (list)
				give_pages(vi, list);
			return -ENOMEM;
363
		}
364
		sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
365

366 367 368 369
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
R
Rusty Russell 已提交
370

371 372 373 374 375 376 377
	first = get_a_page(vi, gfp);
	if (!first) {
		give_pages(vi, list);
		return -ENOMEM;
	}
	p = page_address(first);

378 379 380
	/* vi->rx_sg[0], vi->rx_sg[1] share the same page */
	/* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
	sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
381

382
	/* vi->rx_sg[1] for data packet, from offset */
383
	offset = sizeof(struct padded_vnet_hdr);
384
	sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
385 386 387

	/* chain first in list head */
	first->private = (unsigned long)list;
388
	err = vi->rvq->vq_ops->add_buf(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
389 390 391 392 393
				       first);
	if (err < 0)
		give_pages(vi, first);

	return err;
R
Rusty Russell 已提交
394 395
}

396
static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
397
{
398
	struct page *page;
399 400
	int err;

401 402 403
	page = get_a_page(vi, gfp);
	if (!page)
		return -ENOMEM;
404

405
	sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
406

407
	err = vi->rvq->vq_ops->add_buf(vi->rvq, vi->rx_sg, 0, 1, page);
408 409
	if (err < 0)
		give_pages(vi, page);
410

411 412
	return err;
}
413

414 415 416 417 418
/* Returns false if we couldn't fill entirely (OOM). */
static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
{
	int err;
	bool oom = false;
419

420 421 422 423 424 425 426
	do {
		if (vi->mergeable_rx_bufs)
			err = add_recvbuf_mergeable(vi, gfp);
		else if (vi->big_packets)
			err = add_recvbuf_big(vi, gfp);
		else
			err = add_recvbuf_small(vi, gfp);
427

428
		if (err < 0) {
429
			oom = true;
430 431
			break;
		}
432
		++vi->num;
433
	} while (err > 0);
434 435 436
	if (unlikely(vi->num > vi->max))
		vi->max = vi->num;
	vi->rvq->vq_ops->kick(vi->rvq);
437
	return !oom;
438 439
}

440
static void skb_recv_done(struct virtqueue *rvq)
R
Rusty Russell 已提交
441 442
{
	struct virtnet_info *vi = rvq->vdev->priv;
443
	/* Schedule NAPI, Suppress further interrupts if successful. */
444
	if (napi_schedule_prep(&vi->napi)) {
445
		rvq->vq_ops->disable_cb(rvq);
446
		__napi_schedule(&vi->napi);
447
	}
R
Rusty Russell 已提交
448 449
}

450 451 452 453 454 455 456
static void refill_work(struct work_struct *work)
{
	struct virtnet_info *vi;
	bool still_empty;

	vi = container_of(work, struct virtnet_info, refill.work);
	napi_disable(&vi->napi);
457
	still_empty = !try_fill_recv(vi, GFP_KERNEL);
458 459 460 461 462 463 464 465
	napi_enable(&vi->napi);

	/* In theory, this can happen: if we don't get any buffers in
	 * we will *never* try to fill again. */
	if (still_empty)
		schedule_delayed_work(&vi->refill, HZ/2);
}

R
Rusty Russell 已提交
466 467 468
static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
469
	void *buf;
R
Rusty Russell 已提交
470 471 472 473
	unsigned int len, received = 0;

again:
	while (received < budget &&
474 475 476
	       (buf = vi->rvq->vq_ops->get_buf(vi->rvq, &len)) != NULL) {
		receive_buf(vi->dev, buf, len);
		--vi->num;
R
Rusty Russell 已提交
477 478 479
		received++;
	}

480 481 482 483
	if (vi->num < vi->max / 2) {
		if (!try_fill_recv(vi, GFP_ATOMIC))
			schedule_delayed_work(&vi->refill, 0);
	}
R
Rusty Russell 已提交
484

485 486
	/* Out of packets? */
	if (received < budget) {
487
		napi_complete(napi);
488 489
		if (unlikely(!vi->rvq->vq_ops->enable_cb(vi->rvq)) &&
		    napi_schedule_prep(napi)) {
490
			vi->rvq->vq_ops->disable_cb(vi->rvq);
491
			__napi_schedule(napi);
R
Rusty Russell 已提交
492
			goto again;
493
		}
R
Rusty Russell 已提交
494 495 496 497 498
	}

	return received;
}

499
static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
R
Rusty Russell 已提交
500 501
{
	struct sk_buff *skb;
502
	unsigned int len, tot_sgs = 0;
R
Rusty Russell 已提交
503 504 505

	while ((skb = vi->svq->vq_ops->get_buf(vi->svq, &len)) != NULL) {
		pr_debug("Sent skb %p\n", skb);
506
		vi->dev->stats.tx_bytes += skb->len;
R
Rusty Russell 已提交
507
		vi->dev->stats.tx_packets++;
508
		tot_sgs += skb_vnet_hdr(skb)->num_sg;
509
		dev_kfree_skb_any(skb);
R
Rusty Russell 已提交
510
	}
511
	return tot_sgs;
R
Rusty Russell 已提交
512 513
}

514
static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
R
Rusty Russell 已提交
515
{
516
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
R
Rusty Russell 已提交
517 518
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;

J
Johannes Berg 已提交
519
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
R
Rusty Russell 已提交
520 521

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
522 523 524
		hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
		hdr->hdr.csum_start = skb->csum_start - skb_headroom(skb);
		hdr->hdr.csum_offset = skb->csum_offset;
R
Rusty Russell 已提交
525
	} else {
526 527
		hdr->hdr.flags = 0;
		hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
R
Rusty Russell 已提交
528 529 530
	}

	if (skb_is_gso(skb)) {
531 532
		hdr->hdr.hdr_len = skb_headlen(skb);
		hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
R
Rusty Russell 已提交
533
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
534
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
R
Rusty Russell 已提交
535
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
536
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
R
Rusty Russell 已提交
537
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
538
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
R
Rusty Russell 已提交
539 540
		else
			BUG();
R
Rusty Russell 已提交
541
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
542
			hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
R
Rusty Russell 已提交
543
	} else {
544 545
		hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
		hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
R
Rusty Russell 已提交
546 547
	}

548
	hdr->mhdr.num_buffers = 0;
549 550 551

	/* Encode metadata header at front. */
	if (vi->mergeable_rx_bufs)
552
		sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
553
	else
554
		sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
555

556 557 558
	hdr->num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
	return vi->svq->vq_ops->add_buf(vi->svq, vi->tx_sg, hdr->num_sg,
					0, skb);
559 560
}

561
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
562 563
{
	struct virtnet_info *vi = netdev_priv(dev);
564
	int capacity;
565 566 567 568

again:
	/* Free up any pending old buffers before queueing new ones. */
	free_old_xmit_skbs(vi);
569

570
	/* Try to transmit */
571 572 573 574 575 576 577 578 579 580 581 582
	capacity = xmit_skb(vi, skb);

	/* This can happen with OOM and indirect buffers. */
	if (unlikely(capacity < 0)) {
		netif_stop_queue(dev);
		dev_warn(&dev->dev, "Unexpected full queue\n");
		if (unlikely(!vi->svq->vq_ops->enable_cb(vi->svq))) {
			vi->svq->vq_ops->disable_cb(vi->svq);
			netif_start_queue(dev);
			goto again;
		}
		return NETDEV_TX_BUSY;
R
Rusty Russell 已提交
583
	}
584
	vi->svq->vq_ops->kick(vi->svq);
585

586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601
	/* Don't wait up for transmitted skbs to be freed. */
	skb_orphan(skb);
	nf_reset(skb);

	/* Apparently nice girls don't return TX_BUSY; stop the queue
	 * before it gets out of hand.  Naturally, this wastes entries. */
	if (capacity < 2+MAX_SKB_FRAGS) {
		netif_stop_queue(dev);
		if (unlikely(!vi->svq->vq_ops->enable_cb(vi->svq))) {
			/* More just got used, free them then recheck. */
			capacity += free_old_xmit_skbs(vi);
			if (capacity >= 2+MAX_SKB_FRAGS) {
				netif_start_queue(dev);
				vi->svq->vq_ops->disable_cb(vi->svq);
			}
		}
602
	}
603 604

	return NETDEV_TX_OK;
R
Rusty Russell 已提交
605 606
}

607 608 609 610 611 612 613 614 615 616
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
	int ret;

	ret = eth_mac_addr(dev, p);
	if (ret)
		return ret;

617 618 619
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
		                  dev->dev_addr, dev->addr_len);
620 621 622 623

	return 0;
}

624 625 626 627 628 629 630 631 632
#ifdef CONFIG_NET_POLL_CONTROLLER
static void virtnet_netpoll(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_schedule(&vi->napi);
}
#endif

R
Rusty Russell 已提交
633 634 635 636 637
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_enable(&vi->napi);
R
Rusty Russell 已提交
638 639 640

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets
641 642
	 * now.  virtnet_poll wants re-enable the queue, so we disable here.
	 * We synchronize against interrupts via NAPI_STATE_SCHED */
643
	if (napi_schedule_prep(&vi->napi)) {
644
		vi->rvq->vq_ops->disable_cb(vi->rvq);
645
		__napi_schedule(&vi->napi);
646
	}
R
Rusty Russell 已提交
647 648 649
	return 0;
}

650 651 652 653 654 655 656 657
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
 * never fail unless improperly formated.
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
				 struct scatterlist *data, int out, int in)
{
658
	struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
659 660 661
	struct virtio_net_ctrl_hdr ctrl;
	virtio_net_ctrl_ack status = ~0;
	unsigned int tmp;
662
	int i;
663

664 665 666
	/* Caller should know better */
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
		(out + in > VIRTNET_SEND_COMMAND_SG_MAX));
667 668 669 670 671 672 673 674 675 676

	out++; /* Add header */
	in++; /* Add return status */

	ctrl.class = class;
	ctrl.cmd = cmd;

	sg_init_table(sg, out + in);

	sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
677 678
	for_each_sg(data, s, out + in - 2, i)
		sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
679 680
	sg_set_buf(&sg[out + in - 1], &status, sizeof(status));

681
	BUG_ON(vi->cvq->vq_ops->add_buf(vi->cvq, sg, out, in, vi) < 0);
682 683 684 685 686 687 688 689 690 691 692 693 694

	vi->cvq->vq_ops->kick(vi->cvq);

	/*
	 * Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
	while (!vi->cvq->vq_ops->get_buf(vi->cvq, &tmp))
		cpu_relax();

	return status == VIRTIO_NET_OK;
}

R
Rusty Russell 已提交
695 696 697 698 699 700 701 702 703
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_disable(&vi->napi);

	return 0;
}

704 705 706 707 708 709 710 711 712 713 714
static int virtnet_set_tx_csum(struct net_device *dev, u32 data)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;

	if (data && !virtio_has_feature(vdev, VIRTIO_NET_F_CSUM))
		return -ENOSYS;

	return ethtool_op_set_tx_hw_csum(dev, data);
}

715 716 717
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
718
	struct scatterlist sg[2];
719
	u8 promisc, allmulti;
720
	struct virtio_net_ctrl_mac *mac_data;
J
Jiri Pirko 已提交
721
	struct netdev_hw_addr *ha;
722
	int uc_count;
723
	int mc_count;
724 725
	void *buf;
	int i;
726 727 728 729 730

	/* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

731 732
	promisc = ((dev->flags & IFF_PROMISC) != 0);
	allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
733

734
	sg_init_one(sg, &promisc, sizeof(promisc));
735 736 737

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_PROMISC,
738
				  sg, 1, 0))
739 740 741
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
			 promisc ? "en" : "dis");

742
	sg_init_one(sg, &allmulti, sizeof(allmulti));
743 744 745

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_ALLMULTI,
746
				  sg, 1, 0))
747 748
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
			 allmulti ? "en" : "dis");
749

750
	uc_count = netdev_uc_count(dev);
751
	mc_count = netdev_mc_count(dev);
752
	/* MAC filter - use one buffer for both lists */
753 754 755
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
756 757 758 759 760
	if (!buf) {
		dev_warn(&dev->dev, "No memory for MAC address buffer\n");
		return;
	}

761 762
	sg_init_table(sg, 2);

763
	/* Store the unicast list and count in the front of the buffer */
764
	mac_data->entries = uc_count;
J
Jiri Pirko 已提交
765
	i = 0;
766
	netdev_for_each_uc_addr(ha, dev)
J
Jiri Pirko 已提交
767
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
768 769

	sg_set_buf(&sg[0], mac_data,
770
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
771 772

	/* multicast list and count fill the end */
773
	mac_data = (void *)&mac_data->macs[uc_count][0];
774

775
	mac_data->entries = mc_count;
776
	i = 0;
777 778
	netdev_for_each_mc_addr(ha, dev)
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
779 780

	sg_set_buf(&sg[1], mac_data,
781
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
782 783 784 785 786 787 788

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
				  VIRTIO_NET_CTRL_MAC_TABLE_SET,
				  sg, 2, 0))
		dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");

	kfree(buf);
789 790
}

791
static void virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
792 793 794 795
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

796
	sg_init_one(&sg, &vid, sizeof(vid));
797 798 799 800 801 802

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
}

803
static void virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
804 805 806 807
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

808
	sg_init_one(&sg, &vid, sizeof(vid));
809 810 811 812 813 814

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
}

815
static const struct ethtool_ops virtnet_ethtool_ops = {
816 817
	.set_tx_csum = virtnet_set_tx_csum,
	.set_sg = ethtool_op_set_sg,
818
	.set_tso = ethtool_op_set_tso,
819
	.set_ufo = ethtool_op_set_ufo,
820
	.get_link = ethtool_op_get_link,
821 822
};

M
Mark McLoughlin 已提交
823 824 825 826 827 828 829 830 831 832 833
#define MIN_MTU 68
#define MAX_MTU 65535

static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
{
	if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
		return -EINVAL;
	dev->mtu = new_mtu;
	return 0;
}

834 835 836 837 838
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
839
	.ndo_set_mac_address = virtnet_set_mac_address,
840
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
841
	.ndo_change_mtu	     = virtnet_change_mtu,
842 843
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
844 845 846 847 848
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = virtnet_netpoll,
#endif
};

849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
static void virtnet_update_status(struct virtnet_info *vi)
{
	u16 v;

	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS))
		return;

	vi->vdev->config->get(vi->vdev,
			      offsetof(struct virtio_net_config, status),
			      &v, sizeof(v));

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
		return;

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
		netif_carrier_on(vi->dev);
		netif_wake_queue(vi->dev);
	} else {
		netif_carrier_off(vi->dev);
		netif_stop_queue(vi->dev);
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	virtnet_update_status(vi);
}

R
Rusty Russell 已提交
884 885 886 887 888
static int virtnet_probe(struct virtio_device *vdev)
{
	int err;
	struct net_device *dev;
	struct virtnet_info *vi;
889 890 891 892
	struct virtqueue *vqs[3];
	vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
	const char *names[] = { "input", "output", "control" };
	int nvqs;
R
Rusty Russell 已提交
893 894 895 896 897 898 899

	/* Allocate ourselves a network device with room for our info */
	dev = alloc_etherdev(sizeof(struct virtnet_info));
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
900
	dev->netdev_ops = &virtnet_netdev;
R
Rusty Russell 已提交
901
	dev->features = NETIF_F_HIGHDMA;
902
	SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
R
Rusty Russell 已提交
903 904 905
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
906
	if (csum && virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
R
Rusty Russell 已提交
907 908
		/* This opens up the world of extra features. */
		dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
909
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
R
Rusty Russell 已提交
910 911 912
			dev->features |= NETIF_F_TSO | NETIF_F_UFO
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
913
		/* Individual feature bits: what can host handle? */
914
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
915
			dev->features |= NETIF_F_TSO;
916
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
917
			dev->features |= NETIF_F_TSO6;
918
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
919
			dev->features |= NETIF_F_TSO_ECN;
920
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
921
			dev->features |= NETIF_F_UFO;
R
Rusty Russell 已提交
922 923 924
	}

	/* Configuration may specify what MAC to use.  Otherwise random. */
925
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
926 927 928
		vdev->config->get(vdev,
				  offsetof(struct virtio_net_config, mac),
				  dev->dev_addr, dev->addr_len);
929
	} else
R
Rusty Russell 已提交
930 931 932 933
		random_ether_addr(dev->dev_addr);

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
934
	netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
R
Rusty Russell 已提交
935 936
	vi->dev = dev;
	vi->vdev = vdev;
937
	vdev->priv = vi;
938
	vi->pages = NULL;
939
	INIT_DELAYED_WORK(&vi->refill, refill_work);
940 941
	sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
	sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
R
Rusty Russell 已提交
942

943
	/* If we can receive ANY GSO packets, we must allocate large ones. */
944 945 946
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
947 948
		vi->big_packets = true;

949 950 951
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

952 953 954 955 956 957
	/* We expect two virtqueues, receive then send,
	 * and optionally control. */
	nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;

	err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
	if (err)
R
Rusty Russell 已提交
958 959
		goto free;

960 961
	vi->rvq = vqs[0];
	vi->svq = vqs[1];
R
Rusty Russell 已提交
962

963
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
964
		vi->cvq = vqs[2];
965 966 967

		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
			dev->features |= NETIF_F_HW_VLAN_FILTER;
968 969
	}

R
Rusty Russell 已提交
970 971 972
	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
973
		goto free_vqs;
R
Rusty Russell 已提交
974
	}
975 976

	/* Last of all, set up some receive buffers. */
977
	try_fill_recv(vi, GFP_KERNEL);
978 979 980 981 982 983 984

	/* If we didn't even get one input buffer, we're useless. */
	if (vi->num == 0) {
		err = -ENOMEM;
		goto unregister;
	}

985 986
	vi->status = VIRTIO_NET_S_LINK_UP;
	virtnet_update_status(vi);
987
	netif_carrier_on(dev);
988

R
Rusty Russell 已提交
989 990 991
	pr_debug("virtnet: registered device %s\n", dev->name);
	return 0;

992 993
unregister:
	unregister_netdev(dev);
994
	cancel_delayed_work_sync(&vi->refill);
995 996
free_vqs:
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
997 998 999 1000 1001
free:
	free_netdev(dev);
	return err;
}

1002 1003 1004
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
S
Shirley Ma 已提交
1005 1006 1007 1008 1009 1010
	while (1) {
		buf = vi->svq->vq_ops->detach_unused_buf(vi->svq);
		if (!buf)
			break;
		dev_kfree_skb(buf);
	}
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
	while (1) {
		buf = vi->rvq->vq_ops->detach_unused_buf(vi->rvq);
		if (!buf)
			break;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		--vi->num;
	}
	BUG_ON(vi->num != 0);
}

1024
static void __devexit virtnet_remove(struct virtio_device *vdev)
R
Rusty Russell 已提交
1025
{
1026
	struct virtnet_info *vi = vdev->priv;
1027

R
Rusty Russell 已提交
1028 1029 1030
	/* Stop all the virtqueues. */
	vdev->config->reset(vdev);

1031

1032
	unregister_netdev(vi->dev);
1033
	cancel_delayed_work_sync(&vi->refill);
S
Shirley Ma 已提交
1034 1035

	/* Free unused buffers in both send and recv, if any. */
1036
	free_unused_bufs(vi);
1037

1038 1039
	vdev->config->del_vqs(vi->vdev);

1040 1041 1042
	while (vi->pages)
		__free_pages(get_a_page(vi, GFP_KERNEL), 0);

1043
	free_netdev(vi->dev);
R
Rusty Russell 已提交
1044 1045 1046 1047 1048 1049 1050
}

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

1051
static unsigned int features[] = {
1052 1053
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
	VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1054
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1055
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1056
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1057
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1058
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
1059 1060
};

1061
static struct virtio_driver virtio_net_driver = {
1062 1063
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
1064 1065 1066 1067 1068
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtnet_probe,
	.remove =	__devexit_p(virtnet_remove),
1069
	.config_changed = virtnet_config_changed,
R
Rusty Russell 已提交
1070 1071 1072 1073
};

static int __init init(void)
{
1074
	return register_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1075 1076 1077 1078
}

static void __exit fini(void)
{
1079
	unregister_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1080 1081 1082 1083 1084 1085 1086
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");