virtio_net.c 27.5 KB
Newer Older
1
/* A network driver using virtio.
R
Rusty Russell 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
22
#include <linux/ethtool.h>
R
Rusty Russell 已提交
23 24 25 26
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/scatterlist.h>
27
#include <linux/if_vlan.h>
R
Rusty Russell 已提交
28

29 30 31
static int napi_weight = 128;
module_param(napi_weight, int, 0444);

R
Rusty Russell 已提交
32 33 34 35
static int csum = 1, gso = 1;
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);

R
Rusty Russell 已提交
36
/* FIXME: MTU in config. */
37
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
38
#define GOOD_COPY_LEN	128
R
Rusty Russell 已提交
39

40
#define VIRTNET_SEND_COMMAND_SG_MAX    2
41

R
Rusty Russell 已提交
42 43 44
struct virtnet_info
{
	struct virtio_device *vdev;
45
	struct virtqueue *rvq, *svq, *cvq;
R
Rusty Russell 已提交
46 47
	struct net_device *dev;
	struct napi_struct napi;
48
	unsigned int status;
R
Rusty Russell 已提交
49 50 51 52

	/* Number of input buffers, and max we've ever had. */
	unsigned int num, max;

53 54 55
	/* I like... big packets and I cannot lie! */
	bool big_packets;

56 57 58
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

59
	/* Send queue. */
R
Rusty Russell 已提交
60
	struct sk_buff_head send;
61

62 63 64
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

65 66
	/* Chain pages by the private ptr. */
	struct page *pages;
R
Rusty Russell 已提交
67 68
};

69 70 71 72 73
struct skb_vnet_hdr {
	union {
		struct virtio_net_hdr hdr;
		struct virtio_net_hdr_mrg_rxbuf mhdr;
	};
74
	unsigned int num_sg;
75 76
};

77 78 79 80 81 82 83 84 85 86
struct padded_vnet_hdr {
	struct virtio_net_hdr hdr;
	/*
	 * virtio_net_hdr should be in a separated sg buffer because of a
	 * QEMU bug, and data sg buffer shares same page with this header sg.
	 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
	 */
	char padding[6];
};

87
static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
R
Rusty Russell 已提交
88
{
89
	return (struct skb_vnet_hdr *)skb->cb;
R
Rusty Russell 已提交
90 91
}

92 93 94 95 96
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
static void give_pages(struct virtnet_info *vi, struct page *page)
97
{
98
	struct page *end;
99

100 101 102 103
	/* Find end of list, sew whole thing into vi->pages. */
	for (end = page; end->private; end = (struct page *)end->private);
	end->private = (unsigned long)vi->pages;
	vi->pages = page;
104 105
}

106 107 108 109
static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
{
	struct page *p = vi->pages;

110
	if (p) {
111
		vi->pages = (struct page *)p->private;
112 113 114
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
115 116 117 118
		p = alloc_page(gfp_mask);
	return p;
}

119
static void skb_xmit_done(struct virtqueue *svq)
R
Rusty Russell 已提交
120
{
121
	struct virtnet_info *vi = svq->vdev->priv;
R
Rusty Russell 已提交
122

123 124
	/* Suppress further interrupts. */
	svq->vq_ops->disable_cb(svq);
125

126
	/* We were probably waiting for more output buffers. */
R
Rusty Russell 已提交
127 128 129
	netif_wake_queue(vi->dev);
}

130 131
static void set_skb_frag(struct sk_buff *skb, struct page *page,
			 unsigned int offset, unsigned int *len)
R
Rusty Russell 已提交
132
{
133 134 135 136 137 138 139 140 141 142 143 144 145
	int i = skb_shinfo(skb)->nr_frags;
	skb_frag_t *f;

	f = &skb_shinfo(skb)->frags[i];
	f->size = min((unsigned)PAGE_SIZE - offset, *len);
	f->page_offset = offset;
	f->page = page;

	skb->data_len += f->size;
	skb->len += f->size;
	skb_shinfo(skb)->nr_frags++;
	*len -= f->size;
}
146

147 148 149 150 151 152 153
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct page *page, unsigned int len)
{
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
	unsigned int copy, hdr_len, offset;
	char *p;
154

155
	p = page_address(page);
156

157 158 159 160
	/* copy small packet so we can reuse these pages for small data */
	skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;
161

162
	hdr = skb_vnet_hdr(skb);
163

164 165 166 167 168 169 170
	if (vi->mergeable_rx_bufs) {
		hdr_len = sizeof hdr->mhdr;
		offset = hdr_len;
	} else {
		hdr_len = sizeof hdr->hdr;
		offset = sizeof(struct padded_vnet_hdr);
	}
171

172
	memcpy(hdr, p, hdr_len);
173

174 175
	len -= hdr_len;
	p += offset;
176

177 178 179 180
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	memcpy(skb_put(skb, copy), p, copy);
181

182 183
	len -= copy;
	offset += copy;
184

185 186 187 188 189
	while (len) {
		set_skb_frag(skb, page, offset, &len);
		page = (struct page *)page->private;
		offset = 0;
	}
190

191 192
	if (page)
		give_pages(vi, page);
193

194 195
	return skb;
}
196

197 198 199 200 201 202 203 204 205 206 207 208 209 210
static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
{
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
	struct page *page;
	int num_buf, i, len;

	num_buf = hdr->mhdr.num_buffers;
	while (--num_buf) {
		i = skb_shinfo(skb)->nr_frags;
		if (i >= MAX_SKB_FRAGS) {
			pr_debug("%s: packet too long\n", skb->dev->name);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
		}
211

212 213 214 215 216 217
		page = vi->rvq->vq_ops->get_buf(vi->rvq, &len);
		if (!page) {
			pr_debug("%s: rx error: %d buffers missing\n",
				 skb->dev->name, hdr->mhdr.num_buffers);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
218
		}
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;

		set_skb_frag(skb, page, 0, &len);

		--vi->num;
	}
	return 0;
}

static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct sk_buff *skb;
	struct page *page;
	struct skb_vnet_hdr *hdr;
235

236 237 238 239 240 241 242 243 244
	if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		return;
	}
245

246 247 248 249 250 251 252 253
	if (!vi->mergeable_rx_bufs && !vi->big_packets) {
		skb = buf;
		len -= sizeof(struct virtio_net_hdr);
		skb_trim(skb, len);
	} else {
		page = buf;
		skb = page_to_skb(vi, page, len);
		if (unlikely(!skb)) {
254
			dev->stats.rx_dropped++;
255 256
			give_pages(vi, page);
			return;
257
		}
258 259 260 261 262
		if (vi->mergeable_rx_bufs)
			if (receive_mergeable(vi, skb)) {
				dev_kfree_skb(skb);
				return;
			}
263
	}
264

265
	hdr = skb_vnet_hdr(skb);
266
	skb->truesize += skb->data_len;
R
Rusty Russell 已提交
267 268 269
	dev->stats.rx_bytes += skb->len;
	dev->stats.rx_packets++;

270
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
R
Rusty Russell 已提交
271
		pr_debug("Needs csum!\n");
272 273 274
		if (!skb_partial_csum_set(skb,
					  hdr->hdr.csum_start,
					  hdr->hdr.csum_offset))
R
Rusty Russell 已提交
275 276 277
			goto frame_err;
	}

278 279 280 281
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

282
	if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
R
Rusty Russell 已提交
283
		pr_debug("GSO!\n");
284
		switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
R
Rusty Russell 已提交
285 286 287 288 289 290 291 292 293 294 295 296
		case VIRTIO_NET_HDR_GSO_TCPV4:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
			break;
		case VIRTIO_NET_HDR_GSO_UDP:
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
			break;
		case VIRTIO_NET_HDR_GSO_TCPV6:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
			break;
		default:
			if (net_ratelimit())
				printk(KERN_WARNING "%s: bad gso type %u.\n",
297
				       dev->name, hdr->hdr.gso_type);
R
Rusty Russell 已提交
298 299 300
			goto frame_err;
		}

301
		if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
R
Rusty Russell 已提交
302 303
			skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;

304
		skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
R
Rusty Russell 已提交
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
		if (skb_shinfo(skb)->gso_size == 0) {
			if (net_ratelimit())
				printk(KERN_WARNING "%s: zero gso size.\n",
				       dev->name);
			goto frame_err;
		}

		/* Header must be checked, and gso_segs computed. */
		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
		skb_shinfo(skb)->gso_segs = 0;
	}

	netif_receive_skb(skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

325
static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
R
Rusty Russell 已提交
326 327
{
	struct sk_buff *skb;
328 329 330
	struct skb_vnet_hdr *hdr;
	struct scatterlist sg[2];
	int err;
331

332 333 334
	skb = netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN);
	if (unlikely(!skb))
		return -ENOMEM;
R
Rusty Russell 已提交
335

336
	skb_put(skb, MAX_PACKET_LEN);
337

338 339
	hdr = skb_vnet_hdr(skb);
	sg_set_buf(sg, &hdr->hdr, sizeof hdr->hdr);
340

341
	skb_to_sgvec(skb, sg + 1, 0, skb->len);
342

343 344 345
	err = vi->rvq->vq_ops->add_buf(vi->rvq, sg, 0, 2, skb);
	if (err < 0)
		dev_kfree_skb(skb);
346

347 348
	return err;
}
349

350 351 352 353 354 355 356 357 358 359 360 361 362 363
static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
{
	struct scatterlist sg[MAX_SKB_FRAGS + 2];
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

	/* page in sg[MAX_SKB_FRAGS + 1] is list tail */
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
		first = get_a_page(vi, gfp);
		if (!first) {
			if (list)
				give_pages(vi, list);
			return -ENOMEM;
364
		}
365
		sg_set_buf(&sg[i], page_address(first), PAGE_SIZE);
366

367 368 369 370
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
R
Rusty Russell 已提交
371

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394
	first = get_a_page(vi, gfp);
	if (!first) {
		give_pages(vi, list);
		return -ENOMEM;
	}
	p = page_address(first);

	/* sg[0], sg[1] share the same page */
	/* a separated sg[0] for  virtio_net_hdr only during to QEMU bug*/
	sg_set_buf(&sg[0], p, sizeof(struct virtio_net_hdr));

	/* sg[1] for data packet, from offset */
	offset = sizeof(struct padded_vnet_hdr);
	sg_set_buf(&sg[1], p + offset, PAGE_SIZE - offset);

	/* chain first in list head */
	first->private = (unsigned long)list;
	err = vi->rvq->vq_ops->add_buf(vi->rvq, sg, 0, MAX_SKB_FRAGS + 2,
				       first);
	if (err < 0)
		give_pages(vi, first);

	return err;
R
Rusty Russell 已提交
395 396
}

397
static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
398
{
399 400
	struct page *page;
	struct scatterlist sg;
401 402
	int err;

403 404 405
	page = get_a_page(vi, gfp);
	if (!page)
		return -ENOMEM;
406

407
	sg_init_one(&sg, page_address(page), PAGE_SIZE);
408

409 410 411
	err = vi->rvq->vq_ops->add_buf(vi->rvq, &sg, 0, 1, page);
	if (err < 0)
		give_pages(vi, page);
412

413 414
	return err;
}
415

416 417 418 419 420
/* Returns false if we couldn't fill entirely (OOM). */
static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
{
	int err;
	bool oom = false;
421

422 423 424 425 426 427 428
	do {
		if (vi->mergeable_rx_bufs)
			err = add_recvbuf_mergeable(vi, gfp);
		else if (vi->big_packets)
			err = add_recvbuf_big(vi, gfp);
		else
			err = add_recvbuf_small(vi, gfp);
429

430
		if (err < 0) {
431
			oom = true;
432 433
			break;
		}
434
		++vi->num;
435
	} while (err > 0);
436 437 438
	if (unlikely(vi->num > vi->max))
		vi->max = vi->num;
	vi->rvq->vq_ops->kick(vi->rvq);
439
	return !oom;
440 441
}

442
static void skb_recv_done(struct virtqueue *rvq)
R
Rusty Russell 已提交
443 444
{
	struct virtnet_info *vi = rvq->vdev->priv;
445
	/* Schedule NAPI, Suppress further interrupts if successful. */
446
	if (napi_schedule_prep(&vi->napi)) {
447
		rvq->vq_ops->disable_cb(rvq);
448
		__napi_schedule(&vi->napi);
449
	}
R
Rusty Russell 已提交
450 451
}

452 453 454 455 456 457 458
static void refill_work(struct work_struct *work)
{
	struct virtnet_info *vi;
	bool still_empty;

	vi = container_of(work, struct virtnet_info, refill.work);
	napi_disable(&vi->napi);
459
	still_empty = !try_fill_recv(vi, GFP_KERNEL);
460 461 462 463 464 465 466 467
	napi_enable(&vi->napi);

	/* In theory, this can happen: if we don't get any buffers in
	 * we will *never* try to fill again. */
	if (still_empty)
		schedule_delayed_work(&vi->refill, HZ/2);
}

R
Rusty Russell 已提交
468 469 470
static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
471
	void *buf;
R
Rusty Russell 已提交
472 473 474 475
	unsigned int len, received = 0;

again:
	while (received < budget &&
476 477 478
	       (buf = vi->rvq->vq_ops->get_buf(vi->rvq, &len)) != NULL) {
		receive_buf(vi->dev, buf, len);
		--vi->num;
R
Rusty Russell 已提交
479 480 481
		received++;
	}

482 483 484 485
	if (vi->num < vi->max / 2) {
		if (!try_fill_recv(vi, GFP_ATOMIC))
			schedule_delayed_work(&vi->refill, 0);
	}
R
Rusty Russell 已提交
486

487 488
	/* Out of packets? */
	if (received < budget) {
489
		napi_complete(napi);
490 491
		if (unlikely(!vi->rvq->vq_ops->enable_cb(vi->rvq)) &&
		    napi_schedule_prep(napi)) {
492
			vi->rvq->vq_ops->disable_cb(vi->rvq);
493
			__napi_schedule(napi);
R
Rusty Russell 已提交
494
			goto again;
495
		}
R
Rusty Russell 已提交
496 497 498 499 500
	}

	return received;
}

501
static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
R
Rusty Russell 已提交
502 503
{
	struct sk_buff *skb;
504
	unsigned int len, tot_sgs = 0;
R
Rusty Russell 已提交
505 506 507 508

	while ((skb = vi->svq->vq_ops->get_buf(vi->svq, &len)) != NULL) {
		pr_debug("Sent skb %p\n", skb);
		__skb_unlink(skb, &vi->send);
509
		vi->dev->stats.tx_bytes += skb->len;
R
Rusty Russell 已提交
510
		vi->dev->stats.tx_packets++;
511
		tot_sgs += skb_vnet_hdr(skb)->num_sg;
512
		dev_kfree_skb_any(skb);
R
Rusty Russell 已提交
513
	}
514
	return tot_sgs;
R
Rusty Russell 已提交
515 516
}

517
static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
R
Rusty Russell 已提交
518
{
519
	struct scatterlist sg[2+MAX_SKB_FRAGS];
520
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
R
Rusty Russell 已提交
521 522
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;

523
	sg_init_table(sg, 2+MAX_SKB_FRAGS);
524

J
Johannes Berg 已提交
525
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
R
Rusty Russell 已提交
526 527

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
528 529 530
		hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
		hdr->hdr.csum_start = skb->csum_start - skb_headroom(skb);
		hdr->hdr.csum_offset = skb->csum_offset;
R
Rusty Russell 已提交
531
	} else {
532 533
		hdr->hdr.flags = 0;
		hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
R
Rusty Russell 已提交
534 535 536
	}

	if (skb_is_gso(skb)) {
537 538
		hdr->hdr.hdr_len = skb_headlen(skb);
		hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
R
Rusty Russell 已提交
539
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
540
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
R
Rusty Russell 已提交
541
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
542
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
R
Rusty Russell 已提交
543
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
544
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
R
Rusty Russell 已提交
545 546
		else
			BUG();
R
Rusty Russell 已提交
547
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
548
			hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
R
Rusty Russell 已提交
549
	} else {
550 551
		hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
		hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
R
Rusty Russell 已提交
552 553
	}

554
	hdr->mhdr.num_buffers = 0;
555 556 557

	/* Encode metadata header at front. */
	if (vi->mergeable_rx_bufs)
558
		sg_set_buf(sg, &hdr->mhdr, sizeof hdr->mhdr);
559
	else
560
		sg_set_buf(sg, &hdr->hdr, sizeof hdr->hdr);
561

562 563
	hdr->num_sg = skb_to_sgvec(skb, sg+1, 0, skb->len) + 1;
	return vi->svq->vq_ops->add_buf(vi->svq, sg, hdr->num_sg, 0, skb);
564 565
}

566
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
567 568
{
	struct virtnet_info *vi = netdev_priv(dev);
569
	int capacity;
570 571 572 573

again:
	/* Free up any pending old buffers before queueing new ones. */
	free_old_xmit_skbs(vi);
574

575
	/* Try to transmit */
576 577 578 579 580 581 582 583 584 585 586 587
	capacity = xmit_skb(vi, skb);

	/* This can happen with OOM and indirect buffers. */
	if (unlikely(capacity < 0)) {
		netif_stop_queue(dev);
		dev_warn(&dev->dev, "Unexpected full queue\n");
		if (unlikely(!vi->svq->vq_ops->enable_cb(vi->svq))) {
			vi->svq->vq_ops->disable_cb(vi->svq);
			netif_start_queue(dev);
			goto again;
		}
		return NETDEV_TX_BUSY;
R
Rusty Russell 已提交
588
	}
589
	vi->svq->vq_ops->kick(vi->svq);
590 591 592 593 594 595 596 597 598 599

	/*
	 * Put new one in send queue.  You'd expect we'd need this before
	 * xmit_skb calls add_buf(), since the callback can be triggered
	 * immediately after that.  But since the callback just triggers
	 * another call back here, normal network xmit locking prevents the
	 * race.
	 */
	__skb_queue_head(&vi->send, skb);

600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
	/* Don't wait up for transmitted skbs to be freed. */
	skb_orphan(skb);
	nf_reset(skb);

	/* Apparently nice girls don't return TX_BUSY; stop the queue
	 * before it gets out of hand.  Naturally, this wastes entries. */
	if (capacity < 2+MAX_SKB_FRAGS) {
		netif_stop_queue(dev);
		if (unlikely(!vi->svq->vq_ops->enable_cb(vi->svq))) {
			/* More just got used, free them then recheck. */
			capacity += free_old_xmit_skbs(vi);
			if (capacity >= 2+MAX_SKB_FRAGS) {
				netif_start_queue(dev);
				vi->svq->vq_ops->disable_cb(vi->svq);
			}
		}
616
	}
617 618

	return NETDEV_TX_OK;
R
Rusty Russell 已提交
619 620
}

621 622 623 624 625 626 627 628 629 630
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
	int ret;

	ret = eth_mac_addr(dev, p);
	if (ret)
		return ret;

631 632 633
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
		                  dev->dev_addr, dev->addr_len);
634 635 636 637

	return 0;
}

638 639 640 641 642 643 644 645 646
#ifdef CONFIG_NET_POLL_CONTROLLER
static void virtnet_netpoll(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_schedule(&vi->napi);
}
#endif

R
Rusty Russell 已提交
647 648 649 650 651
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_enable(&vi->napi);
R
Rusty Russell 已提交
652 653 654

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets
655 656
	 * now.  virtnet_poll wants re-enable the queue, so we disable here.
	 * We synchronize against interrupts via NAPI_STATE_SCHED */
657
	if (napi_schedule_prep(&vi->napi)) {
658
		vi->rvq->vq_ops->disable_cb(vi->rvq);
659
		__napi_schedule(&vi->napi);
660
	}
R
Rusty Russell 已提交
661 662 663
	return 0;
}

664 665 666 667 668 669 670 671
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
 * never fail unless improperly formated.
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
				 struct scatterlist *data, int out, int in)
{
672
	struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
673 674 675
	struct virtio_net_ctrl_hdr ctrl;
	virtio_net_ctrl_ack status = ~0;
	unsigned int tmp;
676
	int i;
677

678 679 680
	/* Caller should know better */
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
		(out + in > VIRTNET_SEND_COMMAND_SG_MAX));
681 682 683 684 685 686 687 688 689 690

	out++; /* Add header */
	in++; /* Add return status */

	ctrl.class = class;
	ctrl.cmd = cmd;

	sg_init_table(sg, out + in);

	sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
691 692
	for_each_sg(data, s, out + in - 2, i)
		sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
693 694
	sg_set_buf(&sg[out + in - 1], &status, sizeof(status));

695
	BUG_ON(vi->cvq->vq_ops->add_buf(vi->cvq, sg, out, in, vi) < 0);
696 697 698 699 700 701 702 703 704 705 706 707 708

	vi->cvq->vq_ops->kick(vi->cvq);

	/*
	 * Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
	while (!vi->cvq->vq_ops->get_buf(vi->cvq, &tmp))
		cpu_relax();

	return status == VIRTIO_NET_OK;
}

R
Rusty Russell 已提交
709 710 711 712 713 714 715 716 717
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_disable(&vi->napi);

	return 0;
}

718 719 720 721 722 723 724 725 726 727 728
static int virtnet_set_tx_csum(struct net_device *dev, u32 data)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;

	if (data && !virtio_has_feature(vdev, VIRTIO_NET_F_CSUM))
		return -ENOSYS;

	return ethtool_op_set_tx_hw_csum(dev, data);
}

729 730 731
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
732
	struct scatterlist sg[2];
733
	u8 promisc, allmulti;
734 735
	struct virtio_net_ctrl_mac *mac_data;
	struct dev_addr_list *addr;
J
Jiri Pirko 已提交
736
	struct netdev_hw_addr *ha;
737
	int uc_count;
738
	int mc_count;
739 740
	void *buf;
	int i;
741 742 743 744 745

	/* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

746 747
	promisc = ((dev->flags & IFF_PROMISC) != 0);
	allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
748

749
	sg_init_one(sg, &promisc, sizeof(promisc));
750 751 752

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_PROMISC,
753
				  sg, 1, 0))
754 755 756
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
			 promisc ? "en" : "dis");

757
	sg_init_one(sg, &allmulti, sizeof(allmulti));
758 759 760

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_ALLMULTI,
761
				  sg, 1, 0))
762 763
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
			 allmulti ? "en" : "dis");
764

765
	uc_count = netdev_uc_count(dev);
766
	mc_count = netdev_mc_count(dev);
767
	/* MAC filter - use one buffer for both lists */
768 769 770
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
771 772 773 774 775
	if (!buf) {
		dev_warn(&dev->dev, "No memory for MAC address buffer\n");
		return;
	}

776 777
	sg_init_table(sg, 2);

778
	/* Store the unicast list and count in the front of the buffer */
779
	mac_data->entries = uc_count;
J
Jiri Pirko 已提交
780
	i = 0;
781
	netdev_for_each_uc_addr(ha, dev)
J
Jiri Pirko 已提交
782
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
783 784

	sg_set_buf(&sg[0], mac_data,
785
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
786 787

	/* multicast list and count fill the end */
788
	mac_data = (void *)&mac_data->macs[uc_count][0];
789

790
	mac_data->entries = mc_count;
791
	addr = dev->mc_list;
792
	for (i = 0; i < mc_count; i++, addr = addr->next)
793 794 795
		memcpy(&mac_data->macs[i][0], addr->da_addr, ETH_ALEN);

	sg_set_buf(&sg[1], mac_data,
796
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
797 798 799 800 801 802 803

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
				  VIRTIO_NET_CTRL_MAC_TABLE_SET,
				  sg, 2, 0))
		dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");

	kfree(buf);
804 805
}

806
static void virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
807 808 809 810
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

811
	sg_init_one(&sg, &vid, sizeof(vid));
812 813 814 815 816 817

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
}

818
static void virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
819 820 821 822
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

823
	sg_init_one(&sg, &vid, sizeof(vid));
824 825 826 827 828 829

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
}

830
static const struct ethtool_ops virtnet_ethtool_ops = {
831 832
	.set_tx_csum = virtnet_set_tx_csum,
	.set_sg = ethtool_op_set_sg,
833
	.set_tso = ethtool_op_set_tso,
834
	.set_ufo = ethtool_op_set_ufo,
835
	.get_link = ethtool_op_get_link,
836 837
};

M
Mark McLoughlin 已提交
838 839 840 841 842 843 844 845 846 847 848
#define MIN_MTU 68
#define MAX_MTU 65535

static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
{
	if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
		return -EINVAL;
	dev->mtu = new_mtu;
	return 0;
}

849 850 851 852 853
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
854
	.ndo_set_mac_address = virtnet_set_mac_address,
855
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
856
	.ndo_change_mtu	     = virtnet_change_mtu,
857 858
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
859 860 861 862 863
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = virtnet_netpoll,
#endif
};

864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898
static void virtnet_update_status(struct virtnet_info *vi)
{
	u16 v;

	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS))
		return;

	vi->vdev->config->get(vi->vdev,
			      offsetof(struct virtio_net_config, status),
			      &v, sizeof(v));

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
		return;

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
		netif_carrier_on(vi->dev);
		netif_wake_queue(vi->dev);
	} else {
		netif_carrier_off(vi->dev);
		netif_stop_queue(vi->dev);
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	virtnet_update_status(vi);
}

R
Rusty Russell 已提交
899 900 901 902 903
static int virtnet_probe(struct virtio_device *vdev)
{
	int err;
	struct net_device *dev;
	struct virtnet_info *vi;
904 905 906 907
	struct virtqueue *vqs[3];
	vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
	const char *names[] = { "input", "output", "control" };
	int nvqs;
R
Rusty Russell 已提交
908 909 910 911 912 913 914

	/* Allocate ourselves a network device with room for our info */
	dev = alloc_etherdev(sizeof(struct virtnet_info));
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
915
	dev->netdev_ops = &virtnet_netdev;
R
Rusty Russell 已提交
916
	dev->features = NETIF_F_HIGHDMA;
917
	SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
R
Rusty Russell 已提交
918 919 920
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
921
	if (csum && virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
R
Rusty Russell 已提交
922 923
		/* This opens up the world of extra features. */
		dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
924
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
R
Rusty Russell 已提交
925 926 927
			dev->features |= NETIF_F_TSO | NETIF_F_UFO
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
928
		/* Individual feature bits: what can host handle? */
929
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
930
			dev->features |= NETIF_F_TSO;
931
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
932
			dev->features |= NETIF_F_TSO6;
933
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
934
			dev->features |= NETIF_F_TSO_ECN;
935
		if (gso && virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
936
			dev->features |= NETIF_F_UFO;
R
Rusty Russell 已提交
937 938 939
	}

	/* Configuration may specify what MAC to use.  Otherwise random. */
940
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
941 942 943
		vdev->config->get(vdev,
				  offsetof(struct virtio_net_config, mac),
				  dev->dev_addr, dev->addr_len);
944
	} else
R
Rusty Russell 已提交
945 946 947 948
		random_ether_addr(dev->dev_addr);

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
949
	netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
R
Rusty Russell 已提交
950 951
	vi->dev = dev;
	vi->vdev = vdev;
952
	vdev->priv = vi;
953
	vi->pages = NULL;
954
	INIT_DELAYED_WORK(&vi->refill, refill_work);
R
Rusty Russell 已提交
955

956
	/* If we can receive ANY GSO packets, we must allocate large ones. */
957 958 959
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
960 961
		vi->big_packets = true;

962 963 964
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

965 966 967 968 969 970
	/* We expect two virtqueues, receive then send,
	 * and optionally control. */
	nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;

	err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
	if (err)
R
Rusty Russell 已提交
971 972
		goto free;

973 974
	vi->rvq = vqs[0];
	vi->svq = vqs[1];
R
Rusty Russell 已提交
975

976
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
977
		vi->cvq = vqs[2];
978 979 980

		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
			dev->features |= NETIF_F_HW_VLAN_FILTER;
981 982
	}

983
	/* Initialize our empty send queue. */
R
Rusty Russell 已提交
984 985 986 987 988
	skb_queue_head_init(&vi->send);

	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
989
		goto free_vqs;
R
Rusty Russell 已提交
990
	}
991 992

	/* Last of all, set up some receive buffers. */
993
	try_fill_recv(vi, GFP_KERNEL);
994 995 996 997 998 999 1000

	/* If we didn't even get one input buffer, we're useless. */
	if (vi->num == 0) {
		err = -ENOMEM;
		goto unregister;
	}

1001 1002
	vi->status = VIRTIO_NET_S_LINK_UP;
	virtnet_update_status(vi);
1003
	netif_carrier_on(dev);
1004

R
Rusty Russell 已提交
1005 1006 1007
	pr_debug("virtnet: registered device %s\n", dev->name);
	return 0;

1008 1009
unregister:
	unregister_netdev(dev);
1010
	cancel_delayed_work_sync(&vi->refill);
1011 1012
free_vqs:
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
1013 1014 1015 1016 1017
free:
	free_netdev(dev);
	return err;
}

1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
	while (1) {
		buf = vi->rvq->vq_ops->detach_unused_buf(vi->rvq);
		if (!buf)
			break;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		--vi->num;
	}
	BUG_ON(vi->num != 0);
}

1034
static void __devexit virtnet_remove(struct virtio_device *vdev)
R
Rusty Russell 已提交
1035
{
1036
	struct virtnet_info *vi = vdev->priv;
1037

R
Rusty Russell 已提交
1038 1039 1040
	/* Stop all the virtqueues. */
	vdev->config->reset(vdev);

1041
	/* Free our skbs in send queue, if any. */
W
Wang Chen 已提交
1042
	__skb_queue_purge(&vi->send);
1043

1044
	unregister_netdev(vi->dev);
1045
	cancel_delayed_work_sync(&vi->refill);
1046
	free_unused_bufs(vi);
1047

1048 1049
	vdev->config->del_vqs(vi->vdev);

1050 1051 1052
	while (vi->pages)
		__free_pages(get_a_page(vi, GFP_KERNEL), 0);

1053
	free_netdev(vi->dev);
R
Rusty Russell 已提交
1054 1055 1056 1057 1058 1059 1060
}

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

1061
static unsigned int features[] = {
1062 1063
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
	VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1064
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1065
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1066
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1067
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1068
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
1069 1070
};

1071
static struct virtio_driver virtio_net_driver = {
1072 1073
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
1074 1075 1076 1077 1078
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtnet_probe,
	.remove =	__devexit_p(virtnet_remove),
1079
	.config_changed = virtnet_config_changed,
R
Rusty Russell 已提交
1080 1081 1082 1083
};

static int __init init(void)
{
1084
	return register_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1085 1086 1087 1088
}

static void __exit fini(void)
{
1089
	unregister_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1090 1091 1092 1093 1094 1095 1096
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");