virtio_net.c 27.2 KB
Newer Older
1
/* A network driver using virtio.
R
Rusty Russell 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
22
#include <linux/ethtool.h>
R
Rusty Russell 已提交
23 24 25 26
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/scatterlist.h>
27
#include <linux/if_vlan.h>
28
#include <linux/slab.h>
R
Rusty Russell 已提交
29

30 31 32
static int napi_weight = 128;
module_param(napi_weight, int, 0444);

R
Rusty Russell 已提交
33 34 35 36
static int csum = 1, gso = 1;
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);

R
Rusty Russell 已提交
37
/* FIXME: MTU in config. */
38
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
39
#define GOOD_COPY_LEN	128
R
Rusty Russell 已提交
40

41
#define VIRTNET_SEND_COMMAND_SG_MAX    2
42

43
struct virtnet_info {
R
Rusty Russell 已提交
44
	struct virtio_device *vdev;
45
	struct virtqueue *rvq, *svq, *cvq;
R
Rusty Russell 已提交
46 47
	struct net_device *dev;
	struct napi_struct napi;
48
	unsigned int status;
R
Rusty Russell 已提交
49 50 51 52

	/* Number of input buffers, and max we've ever had. */
	unsigned int num, max;

53 54 55
	/* I like... big packets and I cannot lie! */
	bool big_packets;

56 57 58
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

59 60 61
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

62 63
	/* Chain pages by the private ptr. */
	struct page *pages;
64 65 66 67

	/* fragments + linear part + virtio header */
	struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
	struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
R
Rusty Russell 已提交
68 69
};

70 71 72 73 74
struct skb_vnet_hdr {
	union {
		struct virtio_net_hdr hdr;
		struct virtio_net_hdr_mrg_rxbuf mhdr;
	};
75
	unsigned int num_sg;
76 77
};

78 79 80 81 82 83 84 85 86 87
struct padded_vnet_hdr {
	struct virtio_net_hdr hdr;
	/*
	 * virtio_net_hdr should be in a separated sg buffer because of a
	 * QEMU bug, and data sg buffer shares same page with this header sg.
	 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
	 */
	char padding[6];
};

88
static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
R
Rusty Russell 已提交
89
{
90
	return (struct skb_vnet_hdr *)skb->cb;
R
Rusty Russell 已提交
91 92
}

93 94 95 96 97
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
static void give_pages(struct virtnet_info *vi, struct page *page)
98
{
99
	struct page *end;
100

101 102 103 104
	/* Find end of list, sew whole thing into vi->pages. */
	for (end = page; end->private; end = (struct page *)end->private);
	end->private = (unsigned long)vi->pages;
	vi->pages = page;
105 106
}

107 108 109 110
static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
{
	struct page *p = vi->pages;

111
	if (p) {
112
		vi->pages = (struct page *)p->private;
113 114 115
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
116 117 118 119
		p = alloc_page(gfp_mask);
	return p;
}

120
static void skb_xmit_done(struct virtqueue *svq)
R
Rusty Russell 已提交
121
{
122
	struct virtnet_info *vi = svq->vdev->priv;
R
Rusty Russell 已提交
123

124
	/* Suppress further interrupts. */
125
	virtqueue_disable_cb(svq);
126

127
	/* We were probably waiting for more output buffers. */
R
Rusty Russell 已提交
128 129 130
	netif_wake_queue(vi->dev);
}

131 132
static void set_skb_frag(struct sk_buff *skb, struct page *page,
			 unsigned int offset, unsigned int *len)
R
Rusty Russell 已提交
133
{
134 135 136 137 138 139 140 141 142 143 144 145 146
	int i = skb_shinfo(skb)->nr_frags;
	skb_frag_t *f;

	f = &skb_shinfo(skb)->frags[i];
	f->size = min((unsigned)PAGE_SIZE - offset, *len);
	f->page_offset = offset;
	f->page = page;

	skb->data_len += f->size;
	skb->len += f->size;
	skb_shinfo(skb)->nr_frags++;
	*len -= f->size;
}
147

148 149 150 151 152 153 154
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct page *page, unsigned int len)
{
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
	unsigned int copy, hdr_len, offset;
	char *p;
155

156
	p = page_address(page);
157

158 159 160 161
	/* copy small packet so we can reuse these pages for small data */
	skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;
162

163
	hdr = skb_vnet_hdr(skb);
164

165 166 167 168 169 170 171
	if (vi->mergeable_rx_bufs) {
		hdr_len = sizeof hdr->mhdr;
		offset = hdr_len;
	} else {
		hdr_len = sizeof hdr->hdr;
		offset = sizeof(struct padded_vnet_hdr);
	}
172

173
	memcpy(hdr, p, hdr_len);
174

175 176
	len -= hdr_len;
	p += offset;
177

178 179 180 181
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	memcpy(skb_put(skb, copy), p, copy);
182

183 184
	len -= copy;
	offset += copy;
185

186 187 188 189 190
	while (len) {
		set_skb_frag(skb, page, offset, &len);
		page = (struct page *)page->private;
		offset = 0;
	}
191

192 193
	if (page)
		give_pages(vi, page);
194

195 196
	return skb;
}
197

198 199 200 201 202 203 204 205 206 207 208 209 210 211
static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
{
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
	struct page *page;
	int num_buf, i, len;

	num_buf = hdr->mhdr.num_buffers;
	while (--num_buf) {
		i = skb_shinfo(skb)->nr_frags;
		if (i >= MAX_SKB_FRAGS) {
			pr_debug("%s: packet too long\n", skb->dev->name);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
		}
212

213
		page = virtqueue_get_buf(vi->rvq, &len);
214 215 216 217 218
		if (!page) {
			pr_debug("%s: rx error: %d buffers missing\n",
				 skb->dev->name, hdr->mhdr.num_buffers);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
219
		}
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;

		set_skb_frag(skb, page, 0, &len);

		--vi->num;
	}
	return 0;
}

static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct sk_buff *skb;
	struct page *page;
	struct skb_vnet_hdr *hdr;
236

237 238 239 240 241 242 243 244 245
	if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		return;
	}
246

247 248 249 250 251 252 253 254
	if (!vi->mergeable_rx_bufs && !vi->big_packets) {
		skb = buf;
		len -= sizeof(struct virtio_net_hdr);
		skb_trim(skb, len);
	} else {
		page = buf;
		skb = page_to_skb(vi, page, len);
		if (unlikely(!skb)) {
255
			dev->stats.rx_dropped++;
256 257
			give_pages(vi, page);
			return;
258
		}
259 260 261 262 263
		if (vi->mergeable_rx_bufs)
			if (receive_mergeable(vi, skb)) {
				dev_kfree_skb(skb);
				return;
			}
264
	}
265

266
	hdr = skb_vnet_hdr(skb);
267
	skb->truesize += skb->data_len;
R
Rusty Russell 已提交
268 269 270
	dev->stats.rx_bytes += skb->len;
	dev->stats.rx_packets++;

271
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
R
Rusty Russell 已提交
272
		pr_debug("Needs csum!\n");
273 274 275
		if (!skb_partial_csum_set(skb,
					  hdr->hdr.csum_start,
					  hdr->hdr.csum_offset))
R
Rusty Russell 已提交
276 277 278
			goto frame_err;
	}

279 280 281 282
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

283
	if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
R
Rusty Russell 已提交
284
		pr_debug("GSO!\n");
285
		switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
R
Rusty Russell 已提交
286 287 288 289 290 291 292 293 294 295 296 297
		case VIRTIO_NET_HDR_GSO_TCPV4:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
			break;
		case VIRTIO_NET_HDR_GSO_UDP:
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
			break;
		case VIRTIO_NET_HDR_GSO_TCPV6:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
			break;
		default:
			if (net_ratelimit())
				printk(KERN_WARNING "%s: bad gso type %u.\n",
298
				       dev->name, hdr->hdr.gso_type);
R
Rusty Russell 已提交
299 300 301
			goto frame_err;
		}

302
		if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
R
Rusty Russell 已提交
303 304
			skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;

305
		skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
R
Rusty Russell 已提交
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
		if (skb_shinfo(skb)->gso_size == 0) {
			if (net_ratelimit())
				printk(KERN_WARNING "%s: zero gso size.\n",
				       dev->name);
			goto frame_err;
		}

		/* Header must be checked, and gso_segs computed. */
		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
		skb_shinfo(skb)->gso_segs = 0;
	}

	netif_receive_skb(skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

326
static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
R
Rusty Russell 已提交
327 328
{
	struct sk_buff *skb;
329 330
	struct skb_vnet_hdr *hdr;
	int err;
331

332 333 334
	skb = netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN);
	if (unlikely(!skb))
		return -ENOMEM;
R
Rusty Russell 已提交
335

336
	skb_put(skb, MAX_PACKET_LEN);
337

338
	hdr = skb_vnet_hdr(skb);
339
	sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
340

341
	skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
342

343
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 2, skb, gfp);
344 345
	if (err < 0)
		dev_kfree_skb(skb);
346

347 348
	return err;
}
349

350 351 352 353 354 355
static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

356
	/* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
357 358 359 360 361 362
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
		first = get_a_page(vi, gfp);
		if (!first) {
			if (list)
				give_pages(vi, list);
			return -ENOMEM;
363
		}
364
		sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
365

366 367 368 369
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
R
Rusty Russell 已提交
370

371 372 373 374 375 376 377
	first = get_a_page(vi, gfp);
	if (!first) {
		give_pages(vi, list);
		return -ENOMEM;
	}
	p = page_address(first);

378 379 380
	/* vi->rx_sg[0], vi->rx_sg[1] share the same page */
	/* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
	sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
381

382
	/* vi->rx_sg[1] for data packet, from offset */
383
	offset = sizeof(struct padded_vnet_hdr);
384
	sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
385 386 387

	/* chain first in list head */
	first->private = (unsigned long)list;
388 389
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
				    first, gfp);
390 391 392 393
	if (err < 0)
		give_pages(vi, first);

	return err;
R
Rusty Russell 已提交
394 395
}

396
static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
397
{
398
	struct page *page;
399 400
	int err;

401 402 403
	page = get_a_page(vi, gfp);
	if (!page)
		return -ENOMEM;
404

405
	sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
406

407
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 1, page, gfp);
408 409
	if (err < 0)
		give_pages(vi, page);
410

411 412
	return err;
}
413

414 415 416 417
/* Returns false if we couldn't fill entirely (OOM). */
static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
{
	int err;
418
	bool oom;
419

420 421 422 423 424 425 426
	do {
		if (vi->mergeable_rx_bufs)
			err = add_recvbuf_mergeable(vi, gfp);
		else if (vi->big_packets)
			err = add_recvbuf_big(vi, gfp);
		else
			err = add_recvbuf_small(vi, gfp);
427

428 429
		oom = err == -ENOMEM;
		if (err < 0)
430
			break;
431
		++vi->num;
432
	} while (err > 0);
433 434
	if (unlikely(vi->num > vi->max))
		vi->max = vi->num;
435
	virtqueue_kick(vi->rvq);
436
	return !oom;
437 438
}

439
static void skb_recv_done(struct virtqueue *rvq)
R
Rusty Russell 已提交
440 441
{
	struct virtnet_info *vi = rvq->vdev->priv;
442
	/* Schedule NAPI, Suppress further interrupts if successful. */
443
	if (napi_schedule_prep(&vi->napi)) {
444
		virtqueue_disable_cb(rvq);
445
		__napi_schedule(&vi->napi);
446
	}
R
Rusty Russell 已提交
447 448
}

449 450 451 452 453 454 455
static void refill_work(struct work_struct *work)
{
	struct virtnet_info *vi;
	bool still_empty;

	vi = container_of(work, struct virtnet_info, refill.work);
	napi_disable(&vi->napi);
456
	still_empty = !try_fill_recv(vi, GFP_KERNEL);
457 458 459 460 461 462 463 464
	napi_enable(&vi->napi);

	/* In theory, this can happen: if we don't get any buffers in
	 * we will *never* try to fill again. */
	if (still_empty)
		schedule_delayed_work(&vi->refill, HZ/2);
}

R
Rusty Russell 已提交
465 466 467
static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
468
	void *buf;
R
Rusty Russell 已提交
469 470 471 472
	unsigned int len, received = 0;

again:
	while (received < budget &&
473
	       (buf = virtqueue_get_buf(vi->rvq, &len)) != NULL) {
474 475
		receive_buf(vi->dev, buf, len);
		--vi->num;
R
Rusty Russell 已提交
476 477 478
		received++;
	}

479 480 481 482
	if (vi->num < vi->max / 2) {
		if (!try_fill_recv(vi, GFP_ATOMIC))
			schedule_delayed_work(&vi->refill, 0);
	}
R
Rusty Russell 已提交
483

484 485
	/* Out of packets? */
	if (received < budget) {
486
		napi_complete(napi);
487
		if (unlikely(!virtqueue_enable_cb(vi->rvq)) &&
488
		    napi_schedule_prep(napi)) {
489
			virtqueue_disable_cb(vi->rvq);
490
			__napi_schedule(napi);
R
Rusty Russell 已提交
491
			goto again;
492
		}
R
Rusty Russell 已提交
493 494 495 496 497
	}

	return received;
}

498
static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
R
Rusty Russell 已提交
499 500
{
	struct sk_buff *skb;
501
	unsigned int len, tot_sgs = 0;
R
Rusty Russell 已提交
502

503
	while ((skb = virtqueue_get_buf(vi->svq, &len)) != NULL) {
R
Rusty Russell 已提交
504
		pr_debug("Sent skb %p\n", skb);
505
		vi->dev->stats.tx_bytes += skb->len;
R
Rusty Russell 已提交
506
		vi->dev->stats.tx_packets++;
507
		tot_sgs += skb_vnet_hdr(skb)->num_sg;
508
		dev_kfree_skb_any(skb);
R
Rusty Russell 已提交
509
	}
510
	return tot_sgs;
R
Rusty Russell 已提交
511 512
}

513
static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
R
Rusty Russell 已提交
514
{
515
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
R
Rusty Russell 已提交
516 517
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;

J
Johannes Berg 已提交
518
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
R
Rusty Russell 已提交
519 520

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
521 522 523
		hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
		hdr->hdr.csum_start = skb->csum_start - skb_headroom(skb);
		hdr->hdr.csum_offset = skb->csum_offset;
R
Rusty Russell 已提交
524
	} else {
525 526
		hdr->hdr.flags = 0;
		hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
R
Rusty Russell 已提交
527 528 529
	}

	if (skb_is_gso(skb)) {
530 531
		hdr->hdr.hdr_len = skb_headlen(skb);
		hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
R
Rusty Russell 已提交
532
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
533
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
R
Rusty Russell 已提交
534
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
535
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
R
Rusty Russell 已提交
536
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
537
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
R
Rusty Russell 已提交
538 539
		else
			BUG();
R
Rusty Russell 已提交
540
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
541
			hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
R
Rusty Russell 已提交
542
	} else {
543 544
		hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
		hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
R
Rusty Russell 已提交
545 546
	}

547
	hdr->mhdr.num_buffers = 0;
548 549 550

	/* Encode metadata header at front. */
	if (vi->mergeable_rx_bufs)
551
		sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
552
	else
553
		sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
554

555
	hdr->num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
556
	return virtqueue_add_buf(vi->svq, vi->tx_sg, hdr->num_sg,
557
					0, skb);
558 559
}

560
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
561 562
{
	struct virtnet_info *vi = netdev_priv(dev);
563
	int capacity;
564 565 566 567

again:
	/* Free up any pending old buffers before queueing new ones. */
	free_old_xmit_skbs(vi);
568

569
	/* Try to transmit */
570 571 572 573 574 575
	capacity = xmit_skb(vi, skb);

	/* This can happen with OOM and indirect buffers. */
	if (unlikely(capacity < 0)) {
		netif_stop_queue(dev);
		dev_warn(&dev->dev, "Unexpected full queue\n");
576 577
		if (unlikely(!virtqueue_enable_cb(vi->svq))) {
			virtqueue_disable_cb(vi->svq);
578 579 580 581
			netif_start_queue(dev);
			goto again;
		}
		return NETDEV_TX_BUSY;
R
Rusty Russell 已提交
582
	}
583
	virtqueue_kick(vi->svq);
584

585 586 587 588 589 590 591 592
	/* Don't wait up for transmitted skbs to be freed. */
	skb_orphan(skb);
	nf_reset(skb);

	/* Apparently nice girls don't return TX_BUSY; stop the queue
	 * before it gets out of hand.  Naturally, this wastes entries. */
	if (capacity < 2+MAX_SKB_FRAGS) {
		netif_stop_queue(dev);
593
		if (unlikely(!virtqueue_enable_cb(vi->svq))) {
594 595 596 597
			/* More just got used, free them then recheck. */
			capacity += free_old_xmit_skbs(vi);
			if (capacity >= 2+MAX_SKB_FRAGS) {
				netif_start_queue(dev);
598
				virtqueue_disable_cb(vi->svq);
599 600
			}
		}
601
	}
602 603

	return NETDEV_TX_OK;
R
Rusty Russell 已提交
604 605
}

606 607 608 609 610 611 612 613 614 615
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
	int ret;

	ret = eth_mac_addr(dev, p);
	if (ret)
		return ret;

616 617 618
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
		                  dev->dev_addr, dev->addr_len);
619 620 621 622

	return 0;
}

623 624 625 626 627 628 629 630 631
#ifdef CONFIG_NET_POLL_CONTROLLER
static void virtnet_netpoll(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_schedule(&vi->napi);
}
#endif

R
Rusty Russell 已提交
632 633 634 635 636
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_enable(&vi->napi);
R
Rusty Russell 已提交
637 638 639

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets
640 641
	 * now.  virtnet_poll wants re-enable the queue, so we disable here.
	 * We synchronize against interrupts via NAPI_STATE_SCHED */
642
	if (napi_schedule_prep(&vi->napi)) {
643
		virtqueue_disable_cb(vi->rvq);
644
		__napi_schedule(&vi->napi);
645
	}
R
Rusty Russell 已提交
646 647 648
	return 0;
}

649 650 651 652 653 654 655 656
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
 * never fail unless improperly formated.
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
				 struct scatterlist *data, int out, int in)
{
657
	struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
658 659 660
	struct virtio_net_ctrl_hdr ctrl;
	virtio_net_ctrl_ack status = ~0;
	unsigned int tmp;
661
	int i;
662

663 664 665
	/* Caller should know better */
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
		(out + in > VIRTNET_SEND_COMMAND_SG_MAX));
666 667 668 669 670 671 672 673 674 675

	out++; /* Add header */
	in++; /* Add return status */

	ctrl.class = class;
	ctrl.cmd = cmd;

	sg_init_table(sg, out + in);

	sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
676 677
	for_each_sg(data, s, out + in - 2, i)
		sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
678 679
	sg_set_buf(&sg[out + in - 1], &status, sizeof(status));

680
	BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi) < 0);
681

682
	virtqueue_kick(vi->cvq);
683 684 685 686 687

	/*
	 * Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
688
	while (!virtqueue_get_buf(vi->cvq, &tmp))
689 690 691 692 693
		cpu_relax();

	return status == VIRTIO_NET_OK;
}

R
Rusty Russell 已提交
694 695 696 697 698 699 700 701 702
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_disable(&vi->napi);

	return 0;
}

703 704 705 706 707 708 709 710 711 712 713
static int virtnet_set_tx_csum(struct net_device *dev, u32 data)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;

	if (data && !virtio_has_feature(vdev, VIRTIO_NET_F_CSUM))
		return -ENOSYS;

	return ethtool_op_set_tx_hw_csum(dev, data);
}

714 715 716
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
717
	struct scatterlist sg[2];
718
	u8 promisc, allmulti;
719
	struct virtio_net_ctrl_mac *mac_data;
J
Jiri Pirko 已提交
720
	struct netdev_hw_addr *ha;
721
	int uc_count;
722
	int mc_count;
723 724
	void *buf;
	int i;
725 726 727 728 729

	/* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

730 731
	promisc = ((dev->flags & IFF_PROMISC) != 0);
	allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
732

733
	sg_init_one(sg, &promisc, sizeof(promisc));
734 735 736

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_PROMISC,
737
				  sg, 1, 0))
738 739 740
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
			 promisc ? "en" : "dis");

741
	sg_init_one(sg, &allmulti, sizeof(allmulti));
742 743 744

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_ALLMULTI,
745
				  sg, 1, 0))
746 747
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
			 allmulti ? "en" : "dis");
748

749
	uc_count = netdev_uc_count(dev);
750
	mc_count = netdev_mc_count(dev);
751
	/* MAC filter - use one buffer for both lists */
752 753 754
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
755 756 757 758 759
	if (!buf) {
		dev_warn(&dev->dev, "No memory for MAC address buffer\n");
		return;
	}

760 761
	sg_init_table(sg, 2);

762
	/* Store the unicast list and count in the front of the buffer */
763
	mac_data->entries = uc_count;
J
Jiri Pirko 已提交
764
	i = 0;
765
	netdev_for_each_uc_addr(ha, dev)
J
Jiri Pirko 已提交
766
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
767 768

	sg_set_buf(&sg[0], mac_data,
769
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
770 771

	/* multicast list and count fill the end */
772
	mac_data = (void *)&mac_data->macs[uc_count][0];
773

774
	mac_data->entries = mc_count;
775
	i = 0;
776 777
	netdev_for_each_mc_addr(ha, dev)
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
778 779

	sg_set_buf(&sg[1], mac_data,
780
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
781 782 783 784 785 786 787

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
				  VIRTIO_NET_CTRL_MAC_TABLE_SET,
				  sg, 2, 0))
		dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");

	kfree(buf);
788 789
}

790
static void virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
791 792 793 794
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

795
	sg_init_one(&sg, &vid, sizeof(vid));
796 797 798 799 800 801

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
}

802
static void virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
803 804 805 806
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

807
	sg_init_one(&sg, &vid, sizeof(vid));
808 809 810 811 812 813

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
}

814
static const struct ethtool_ops virtnet_ethtool_ops = {
815 816
	.set_tx_csum = virtnet_set_tx_csum,
	.set_sg = ethtool_op_set_sg,
817
	.set_tso = ethtool_op_set_tso,
818
	.set_ufo = ethtool_op_set_ufo,
819
	.get_link = ethtool_op_get_link,
820 821
};

M
Mark McLoughlin 已提交
822 823 824 825 826 827 828 829 830 831 832
#define MIN_MTU 68
#define MAX_MTU 65535

static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
{
	if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
		return -EINVAL;
	dev->mtu = new_mtu;
	return 0;
}

833 834 835 836 837
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
838
	.ndo_set_mac_address = virtnet_set_mac_address,
839
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
840
	.ndo_change_mtu	     = virtnet_change_mtu,
841 842
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
843 844 845 846 847
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = virtnet_netpoll,
#endif
};

848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
static void virtnet_update_status(struct virtnet_info *vi)
{
	u16 v;

	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS))
		return;

	vi->vdev->config->get(vi->vdev,
			      offsetof(struct virtio_net_config, status),
			      &v, sizeof(v));

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
		return;

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
		netif_carrier_on(vi->dev);
		netif_wake_queue(vi->dev);
	} else {
		netif_carrier_off(vi->dev);
		netif_stop_queue(vi->dev);
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	virtnet_update_status(vi);
}

R
Rusty Russell 已提交
883 884 885 886 887
static int virtnet_probe(struct virtio_device *vdev)
{
	int err;
	struct net_device *dev;
	struct virtnet_info *vi;
888 889 890 891
	struct virtqueue *vqs[3];
	vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
	const char *names[] = { "input", "output", "control" };
	int nvqs;
R
Rusty Russell 已提交
892 893 894 895 896 897 898

	/* Allocate ourselves a network device with room for our info */
	dev = alloc_etherdev(sizeof(struct virtnet_info));
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
899
	dev->netdev_ops = &virtnet_netdev;
R
Rusty Russell 已提交
900
	dev->features = NETIF_F_HIGHDMA;
901
	SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
R
Rusty Russell 已提交
902 903 904
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
905
	if (csum && virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
R
Rusty Russell 已提交
906 907
		/* This opens up the world of extra features. */
		dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
908
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
R
Rusty Russell 已提交
909 910 911
			dev->features |= NETIF_F_TSO | NETIF_F_UFO
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
912
		/* Individual feature bits: what can host handle? */
913
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
914
			dev->features |= NETIF_F_TSO;
915
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
916
			dev->features |= NETIF_F_TSO6;
917
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
918
			dev->features |= NETIF_F_TSO_ECN;
919
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
920
			dev->features |= NETIF_F_UFO;
R
Rusty Russell 已提交
921 922 923
	}

	/* Configuration may specify what MAC to use.  Otherwise random. */
924
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
925 926 927
		vdev->config->get(vdev,
				  offsetof(struct virtio_net_config, mac),
				  dev->dev_addr, dev->addr_len);
928
	} else
R
Rusty Russell 已提交
929 930 931 932
		random_ether_addr(dev->dev_addr);

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
933
	netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
R
Rusty Russell 已提交
934 935
	vi->dev = dev;
	vi->vdev = vdev;
936
	vdev->priv = vi;
937
	vi->pages = NULL;
938
	INIT_DELAYED_WORK(&vi->refill, refill_work);
939 940
	sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
	sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
R
Rusty Russell 已提交
941

942
	/* If we can receive ANY GSO packets, we must allocate large ones. */
943 944 945
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
946 947
		vi->big_packets = true;

948 949 950
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

951 952 953 954 955 956
	/* We expect two virtqueues, receive then send,
	 * and optionally control. */
	nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;

	err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
	if (err)
R
Rusty Russell 已提交
957 958
		goto free;

959 960
	vi->rvq = vqs[0];
	vi->svq = vqs[1];
R
Rusty Russell 已提交
961

962
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
963
		vi->cvq = vqs[2];
964 965 966

		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
			dev->features |= NETIF_F_HW_VLAN_FILTER;
967 968
	}

R
Rusty Russell 已提交
969 970 971
	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
972
		goto free_vqs;
R
Rusty Russell 已提交
973
	}
974 975

	/* Last of all, set up some receive buffers. */
976
	try_fill_recv(vi, GFP_KERNEL);
977 978 979 980 981 982 983

	/* If we didn't even get one input buffer, we're useless. */
	if (vi->num == 0) {
		err = -ENOMEM;
		goto unregister;
	}

984 985
	vi->status = VIRTIO_NET_S_LINK_UP;
	virtnet_update_status(vi);
986
	netif_carrier_on(dev);
987

R
Rusty Russell 已提交
988 989 990
	pr_debug("virtnet: registered device %s\n", dev->name);
	return 0;

991 992
unregister:
	unregister_netdev(dev);
993
	cancel_delayed_work_sync(&vi->refill);
994 995
free_vqs:
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
996 997 998 999 1000
free:
	free_netdev(dev);
	return err;
}

1001 1002 1003
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
S
Shirley Ma 已提交
1004
	while (1) {
1005
		buf = virtqueue_detach_unused_buf(vi->svq);
S
Shirley Ma 已提交
1006 1007 1008 1009
		if (!buf)
			break;
		dev_kfree_skb(buf);
	}
1010
	while (1) {
1011
		buf = virtqueue_detach_unused_buf(vi->rvq);
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
		if (!buf)
			break;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		--vi->num;
	}
	BUG_ON(vi->num != 0);
}

1023
static void __devexit virtnet_remove(struct virtio_device *vdev)
R
Rusty Russell 已提交
1024
{
1025
	struct virtnet_info *vi = vdev->priv;
1026

R
Rusty Russell 已提交
1027 1028 1029
	/* Stop all the virtqueues. */
	vdev->config->reset(vdev);

1030

1031
	unregister_netdev(vi->dev);
1032
	cancel_delayed_work_sync(&vi->refill);
S
Shirley Ma 已提交
1033 1034

	/* Free unused buffers in both send and recv, if any. */
1035
	free_unused_bufs(vi);
1036

1037 1038
	vdev->config->del_vqs(vi->vdev);

1039 1040 1041
	while (vi->pages)
		__free_pages(get_a_page(vi, GFP_KERNEL), 0);

1042
	free_netdev(vi->dev);
R
Rusty Russell 已提交
1043 1044 1045 1046 1047 1048 1049
}

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

1050
static unsigned int features[] = {
1051 1052
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
	VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1053
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1054
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1055
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1056
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1057
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
1058 1059
};

1060
static struct virtio_driver virtio_net_driver = {
1061 1062
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
1063 1064 1065 1066 1067
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtnet_probe,
	.remove =	__devexit_p(virtnet_remove),
1068
	.config_changed = virtnet_config_changed,
R
Rusty Russell 已提交
1069 1070 1071 1072
};

static int __init init(void)
{
1073
	return register_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1074 1075 1076 1077
}

static void __exit fini(void)
{
1078
	unregister_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1079 1080 1081 1082 1083 1084 1085
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");