virtio_net.c 29.0 KB
Newer Older
1
/* A network driver using virtio.
R
Rusty Russell 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
22
#include <linux/ethtool.h>
R
Rusty Russell 已提交
23 24 25 26
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/scatterlist.h>
27
#include <linux/if_vlan.h>
28
#include <linux/slab.h>
R
Rusty Russell 已提交
29

30 31 32
static int napi_weight = 128;
module_param(napi_weight, int, 0444);

R
Rusty Russell 已提交
33 34 35 36
static int csum = 1, gso = 1;
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);

R
Rusty Russell 已提交
37
/* FIXME: MTU in config. */
38
#define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
39
#define GOOD_COPY_LEN	128
R
Rusty Russell 已提交
40

41
#define VIRTNET_SEND_COMMAND_SG_MAX    2
42

43 44 45 46 47 48 49 50 51
struct virtnet_stats {
	struct u64_stats_sync syncp;
	u64 tx_bytes;
	u64 tx_packets;

	u64 rx_bytes;
	u64 rx_packets;
};

52
struct virtnet_info {
R
Rusty Russell 已提交
53
	struct virtio_device *vdev;
54
	struct virtqueue *rvq, *svq, *cvq;
R
Rusty Russell 已提交
55 56
	struct net_device *dev;
	struct napi_struct napi;
57
	unsigned int status;
R
Rusty Russell 已提交
58 59 60 61

	/* Number of input buffers, and max we've ever had. */
	unsigned int num, max;

62 63 64
	/* I like... big packets and I cannot lie! */
	bool big_packets;

65 66 67
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

68 69 70
	/* Active statistics */
	struct virtnet_stats __percpu *stats;

71 72 73
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

74 75
	/* Chain pages by the private ptr. */
	struct page *pages;
76 77 78 79

	/* fragments + linear part + virtio header */
	struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
	struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
R
Rusty Russell 已提交
80 81
};

82 83 84 85 86
struct skb_vnet_hdr {
	union {
		struct virtio_net_hdr hdr;
		struct virtio_net_hdr_mrg_rxbuf mhdr;
	};
87
	unsigned int num_sg;
88 89
};

90 91 92 93 94 95 96 97 98 99
struct padded_vnet_hdr {
	struct virtio_net_hdr hdr;
	/*
	 * virtio_net_hdr should be in a separated sg buffer because of a
	 * QEMU bug, and data sg buffer shares same page with this header sg.
	 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
	 */
	char padding[6];
};

100
static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
R
Rusty Russell 已提交
101
{
102
	return (struct skb_vnet_hdr *)skb->cb;
R
Rusty Russell 已提交
103 104
}

105 106 107 108 109
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
static void give_pages(struct virtnet_info *vi, struct page *page)
110
{
111
	struct page *end;
112

113 114 115 116
	/* Find end of list, sew whole thing into vi->pages. */
	for (end = page; end->private; end = (struct page *)end->private);
	end->private = (unsigned long)vi->pages;
	vi->pages = page;
117 118
}

119 120 121 122
static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
{
	struct page *p = vi->pages;

123
	if (p) {
124
		vi->pages = (struct page *)p->private;
125 126 127
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
128 129 130 131
		p = alloc_page(gfp_mask);
	return p;
}

132
static void skb_xmit_done(struct virtqueue *svq)
R
Rusty Russell 已提交
133
{
134
	struct virtnet_info *vi = svq->vdev->priv;
R
Rusty Russell 已提交
135

136
	/* Suppress further interrupts. */
137
	virtqueue_disable_cb(svq);
138

139
	/* We were probably waiting for more output buffers. */
R
Rusty Russell 已提交
140 141 142
	netif_wake_queue(vi->dev);
}

143 144
static void set_skb_frag(struct sk_buff *skb, struct page *page,
			 unsigned int offset, unsigned int *len)
R
Rusty Russell 已提交
145
{
146 147 148 149 150 151
	int i = skb_shinfo(skb)->nr_frags;
	skb_frag_t *f;

	f = &skb_shinfo(skb)->frags[i];
	f->size = min((unsigned)PAGE_SIZE - offset, *len);
	f->page_offset = offset;
152
	__skb_frag_set_page(f, page);
153 154 155 156 157 158

	skb->data_len += f->size;
	skb->len += f->size;
	skb_shinfo(skb)->nr_frags++;
	*len -= f->size;
}
159

160 161 162 163 164 165 166
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
				   struct page *page, unsigned int len)
{
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
	unsigned int copy, hdr_len, offset;
	char *p;
167

168
	p = page_address(page);
169

170 171 172 173
	/* copy small packet so we can reuse these pages for small data */
	skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;
174

175
	hdr = skb_vnet_hdr(skb);
176

177 178 179 180 181 182 183
	if (vi->mergeable_rx_bufs) {
		hdr_len = sizeof hdr->mhdr;
		offset = hdr_len;
	} else {
		hdr_len = sizeof hdr->hdr;
		offset = sizeof(struct padded_vnet_hdr);
	}
184

185
	memcpy(hdr, p, hdr_len);
186

187 188
	len -= hdr_len;
	p += offset;
189

190 191 192 193
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	memcpy(skb_put(skb, copy), p, copy);
194

195 196
	len -= copy;
	offset += copy;
197

198 199 200 201 202
	while (len) {
		set_skb_frag(skb, page, offset, &len);
		page = (struct page *)page->private;
		offset = 0;
	}
203

204 205
	if (page)
		give_pages(vi, page);
206

207 208
	return skb;
}
209

210 211 212 213 214 215 216 217 218 219 220 221 222 223
static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
{
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
	struct page *page;
	int num_buf, i, len;

	num_buf = hdr->mhdr.num_buffers;
	while (--num_buf) {
		i = skb_shinfo(skb)->nr_frags;
		if (i >= MAX_SKB_FRAGS) {
			pr_debug("%s: packet too long\n", skb->dev->name);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
		}
224
		page = virtqueue_get_buf(vi->rvq, &len);
225 226 227 228 229
		if (!page) {
			pr_debug("%s: rx error: %d buffers missing\n",
				 skb->dev->name, hdr->mhdr.num_buffers);
			skb->dev->stats.rx_length_errors++;
			return -EINVAL;
230
		}
231

232 233 234 235 236 237 238 239 240 241 242 243 244
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;

		set_skb_frag(skb, page, 0, &len);

		--vi->num;
	}
	return 0;
}

static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
{
	struct virtnet_info *vi = netdev_priv(dev);
245
	struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
246 247 248
	struct sk_buff *skb;
	struct page *page;
	struct skb_vnet_hdr *hdr;
249

250 251 252 253 254 255 256 257 258
	if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		return;
	}
259

260 261 262 263 264 265 266 267
	if (!vi->mergeable_rx_bufs && !vi->big_packets) {
		skb = buf;
		len -= sizeof(struct virtio_net_hdr);
		skb_trim(skb, len);
	} else {
		page = buf;
		skb = page_to_skb(vi, page, len);
		if (unlikely(!skb)) {
268
			dev->stats.rx_dropped++;
269 270
			give_pages(vi, page);
			return;
271
		}
272 273 274 275 276
		if (vi->mergeable_rx_bufs)
			if (receive_mergeable(vi, skb)) {
				dev_kfree_skb(skb);
				return;
			}
277
	}
278

279
	hdr = skb_vnet_hdr(skb);
280
	skb->truesize += skb->data_len;
281 282 283 284 285

	u64_stats_update_begin(&stats->syncp);
	stats->rx_bytes += skb->len;
	stats->rx_packets++;
	u64_stats_update_end(&stats->syncp);
R
Rusty Russell 已提交
286

287
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
R
Rusty Russell 已提交
288
		pr_debug("Needs csum!\n");
289 290 291
		if (!skb_partial_csum_set(skb,
					  hdr->hdr.csum_start,
					  hdr->hdr.csum_offset))
R
Rusty Russell 已提交
292
			goto frame_err;
293 294
	} else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
		skb->ip_summed = CHECKSUM_UNNECESSARY;
R
Rusty Russell 已提交
295 296
	}

297 298 299 300
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

301
	if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
R
Rusty Russell 已提交
302
		pr_debug("GSO!\n");
303
		switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
R
Rusty Russell 已提交
304 305 306 307 308 309 310 311 312 313 314 315
		case VIRTIO_NET_HDR_GSO_TCPV4:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
			break;
		case VIRTIO_NET_HDR_GSO_UDP:
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
			break;
		case VIRTIO_NET_HDR_GSO_TCPV6:
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
			break;
		default:
			if (net_ratelimit())
				printk(KERN_WARNING "%s: bad gso type %u.\n",
316
				       dev->name, hdr->hdr.gso_type);
R
Rusty Russell 已提交
317 318 319
			goto frame_err;
		}

320
		if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
R
Rusty Russell 已提交
321 322
			skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;

323
		skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
R
Rusty Russell 已提交
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
		if (skb_shinfo(skb)->gso_size == 0) {
			if (net_ratelimit())
				printk(KERN_WARNING "%s: zero gso size.\n",
				       dev->name);
			goto frame_err;
		}

		/* Header must be checked, and gso_segs computed. */
		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
		skb_shinfo(skb)->gso_segs = 0;
	}

	netif_receive_skb(skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

344
static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
R
Rusty Russell 已提交
345 346
{
	struct sk_buff *skb;
347 348
	struct skb_vnet_hdr *hdr;
	int err;
349

350 351 352
	skb = netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN);
	if (unlikely(!skb))
		return -ENOMEM;
R
Rusty Russell 已提交
353

354
	skb_put(skb, MAX_PACKET_LEN);
355

356
	hdr = skb_vnet_hdr(skb);
357
	sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
358

359
	skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
360

361
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 2, skb, gfp);
362 363
	if (err < 0)
		dev_kfree_skb(skb);
364

365 366
	return err;
}
367

368 369 370 371 372 373
static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

374
	/* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
375 376 377 378 379 380
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
		first = get_a_page(vi, gfp);
		if (!first) {
			if (list)
				give_pages(vi, list);
			return -ENOMEM;
381
		}
382
		sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
383

384 385 386 387
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
R
Rusty Russell 已提交
388

389 390 391 392 393 394 395
	first = get_a_page(vi, gfp);
	if (!first) {
		give_pages(vi, list);
		return -ENOMEM;
	}
	p = page_address(first);

396 397 398
	/* vi->rx_sg[0], vi->rx_sg[1] share the same page */
	/* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
	sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
399

400
	/* vi->rx_sg[1] for data packet, from offset */
401
	offset = sizeof(struct padded_vnet_hdr);
402
	sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
403 404 405

	/* chain first in list head */
	first->private = (unsigned long)list;
406 407
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
				    first, gfp);
408 409 410 411
	if (err < 0)
		give_pages(vi, first);

	return err;
R
Rusty Russell 已提交
412 413
}

414
static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
415
{
416
	struct page *page;
417 418
	int err;

419 420 421
	page = get_a_page(vi, gfp);
	if (!page)
		return -ENOMEM;
422

423
	sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
424

425
	err = virtqueue_add_buf_gfp(vi->rvq, vi->rx_sg, 0, 1, page, gfp);
426 427
	if (err < 0)
		give_pages(vi, page);
428

429 430
	return err;
}
431

432 433 434 435
/* Returns false if we couldn't fill entirely (OOM). */
static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
{
	int err;
436
	bool oom;
437

438 439 440 441 442 443 444
	do {
		if (vi->mergeable_rx_bufs)
			err = add_recvbuf_mergeable(vi, gfp);
		else if (vi->big_packets)
			err = add_recvbuf_big(vi, gfp);
		else
			err = add_recvbuf_small(vi, gfp);
445

446 447
		oom = err == -ENOMEM;
		if (err < 0)
448
			break;
449
		++vi->num;
450
	} while (err > 0);
451 452
	if (unlikely(vi->num > vi->max))
		vi->max = vi->num;
453
	virtqueue_kick(vi->rvq);
454
	return !oom;
455 456
}

457
static void skb_recv_done(struct virtqueue *rvq)
R
Rusty Russell 已提交
458 459
{
	struct virtnet_info *vi = rvq->vdev->priv;
460
	/* Schedule NAPI, Suppress further interrupts if successful. */
461
	if (napi_schedule_prep(&vi->napi)) {
462
		virtqueue_disable_cb(rvq);
463
		__napi_schedule(&vi->napi);
464
	}
R
Rusty Russell 已提交
465 466
}

467 468 469 470 471 472 473 474 475 476 477 478 479 480
static void virtnet_napi_enable(struct virtnet_info *vi)
{
	napi_enable(&vi->napi);

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets
	 * now.  virtnet_poll wants re-enable the queue, so we disable here.
	 * We synchronize against interrupts via NAPI_STATE_SCHED */
	if (napi_schedule_prep(&vi->napi)) {
		virtqueue_disable_cb(vi->rvq);
		__napi_schedule(&vi->napi);
	}
}

481 482 483 484 485 486 487
static void refill_work(struct work_struct *work)
{
	struct virtnet_info *vi;
	bool still_empty;

	vi = container_of(work, struct virtnet_info, refill.work);
	napi_disable(&vi->napi);
488
	still_empty = !try_fill_recv(vi, GFP_KERNEL);
489
	virtnet_napi_enable(vi);
490 491 492 493 494 495 496

	/* In theory, this can happen: if we don't get any buffers in
	 * we will *never* try to fill again. */
	if (still_empty)
		schedule_delayed_work(&vi->refill, HZ/2);
}

R
Rusty Russell 已提交
497 498 499
static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
500
	void *buf;
R
Rusty Russell 已提交
501 502 503 504
	unsigned int len, received = 0;

again:
	while (received < budget &&
505
	       (buf = virtqueue_get_buf(vi->rvq, &len)) != NULL) {
506 507
		receive_buf(vi->dev, buf, len);
		--vi->num;
R
Rusty Russell 已提交
508 509 510
		received++;
	}

511 512 513 514
	if (vi->num < vi->max / 2) {
		if (!try_fill_recv(vi, GFP_ATOMIC))
			schedule_delayed_work(&vi->refill, 0);
	}
R
Rusty Russell 已提交
515

516 517
	/* Out of packets? */
	if (received < budget) {
518
		napi_complete(napi);
519
		if (unlikely(!virtqueue_enable_cb(vi->rvq)) &&
520
		    napi_schedule_prep(napi)) {
521
			virtqueue_disable_cb(vi->rvq);
522
			__napi_schedule(napi);
R
Rusty Russell 已提交
523
			goto again;
524
		}
R
Rusty Russell 已提交
525 526 527 528 529
	}

	return received;
}

530
static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
R
Rusty Russell 已提交
531 532
{
	struct sk_buff *skb;
533
	unsigned int len, tot_sgs = 0;
534
	struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
R
Rusty Russell 已提交
535

536
	while ((skb = virtqueue_get_buf(vi->svq, &len)) != NULL) {
R
Rusty Russell 已提交
537
		pr_debug("Sent skb %p\n", skb);
538 539 540 541 542 543

		u64_stats_update_begin(&stats->syncp);
		stats->tx_bytes += skb->len;
		stats->tx_packets++;
		u64_stats_update_end(&stats->syncp);

544
		tot_sgs += skb_vnet_hdr(skb)->num_sg;
545
		dev_kfree_skb_any(skb);
R
Rusty Russell 已提交
546
	}
547
	return tot_sgs;
R
Rusty Russell 已提交
548 549
}

550
static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
R
Rusty Russell 已提交
551
{
552
	struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
R
Rusty Russell 已提交
553 554
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;

J
Johannes Berg 已提交
555
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
R
Rusty Russell 已提交
556 557

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
558
		hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
559
		hdr->hdr.csum_start = skb_checksum_start_offset(skb);
560
		hdr->hdr.csum_offset = skb->csum_offset;
R
Rusty Russell 已提交
561
	} else {
562 563
		hdr->hdr.flags = 0;
		hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
R
Rusty Russell 已提交
564 565 566
	}

	if (skb_is_gso(skb)) {
567 568
		hdr->hdr.hdr_len = skb_headlen(skb);
		hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
R
Rusty Russell 已提交
569
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
570
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
R
Rusty Russell 已提交
571
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
572
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
R
Rusty Russell 已提交
573
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
574
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
R
Rusty Russell 已提交
575 576
		else
			BUG();
R
Rusty Russell 已提交
577
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
578
			hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
R
Rusty Russell 已提交
579
	} else {
580 581
		hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
		hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
R
Rusty Russell 已提交
582 583
	}

584
	hdr->mhdr.num_buffers = 0;
585 586 587

	/* Encode metadata header at front. */
	if (vi->mergeable_rx_bufs)
588
		sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
589
	else
590
		sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
591

592
	hdr->num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
593
	return virtqueue_add_buf(vi->svq, vi->tx_sg, hdr->num_sg,
594
					0, skb);
595 596
}

597
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
598 599
{
	struct virtnet_info *vi = netdev_priv(dev);
600
	int capacity;
601 602 603

	/* Free up any pending old buffers before queueing new ones. */
	free_old_xmit_skbs(vi);
604

605
	/* Try to transmit */
606 607 608 609
	capacity = xmit_skb(vi, skb);

	/* This can happen with OOM and indirect buffers. */
	if (unlikely(capacity < 0)) {
610 611 612 613 614 615 616 617 618 619
		if (net_ratelimit()) {
			if (likely(capacity == -ENOMEM)) {
				dev_warn(&dev->dev,
					 "TX queue failure: out of memory\n");
			} else {
				dev->stats.tx_fifo_errors++;
				dev_warn(&dev->dev,
					 "Unexpected TX queue failure: %d\n",
					 capacity);
			}
620
		}
621 622 623
		dev->stats.tx_dropped++;
		kfree_skb(skb);
		return NETDEV_TX_OK;
R
Rusty Russell 已提交
624
	}
625
	virtqueue_kick(vi->svq);
626

627 628 629 630 631 632 633 634
	/* Don't wait up for transmitted skbs to be freed. */
	skb_orphan(skb);
	nf_reset(skb);

	/* Apparently nice girls don't return TX_BUSY; stop the queue
	 * before it gets out of hand.  Naturally, this wastes entries. */
	if (capacity < 2+MAX_SKB_FRAGS) {
		netif_stop_queue(dev);
635
		if (unlikely(!virtqueue_enable_cb_delayed(vi->svq))) {
636 637 638 639
			/* More just got used, free them then recheck. */
			capacity += free_old_xmit_skbs(vi);
			if (capacity >= 2+MAX_SKB_FRAGS) {
				netif_start_queue(dev);
640
				virtqueue_disable_cb(vi->svq);
641 642
			}
		}
643
	}
644 645

	return NETDEV_TX_OK;
R
Rusty Russell 已提交
646 647
}

648 649 650 651 652 653 654 655 656 657
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
	int ret;

	ret = eth_mac_addr(dev, p);
	if (ret)
		return ret;

658 659 660
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
		                  dev->dev_addr, dev->addr_len);
661 662 663 664

	return 0;
}

665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698
static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
					       struct rtnl_link_stats64 *tot)
{
	struct virtnet_info *vi = netdev_priv(dev);
	int cpu;
	unsigned int start;

	for_each_possible_cpu(cpu) {
		struct virtnet_stats __percpu *stats
			= per_cpu_ptr(vi->stats, cpu);
		u64 tpackets, tbytes, rpackets, rbytes;

		do {
			start = u64_stats_fetch_begin(&stats->syncp);
			tpackets = stats->tx_packets;
			tbytes   = stats->tx_bytes;
			rpackets = stats->rx_packets;
			rbytes   = stats->rx_bytes;
		} while (u64_stats_fetch_retry(&stats->syncp, start));

		tot->rx_packets += rpackets;
		tot->tx_packets += tpackets;
		tot->rx_bytes   += rbytes;
		tot->tx_bytes   += tbytes;
	}

	tot->tx_dropped = dev->stats.tx_dropped;
	tot->rx_dropped = dev->stats.rx_dropped;
	tot->rx_length_errors = dev->stats.rx_length_errors;
	tot->rx_frame_errors = dev->stats.rx_frame_errors;

	return tot;
}

699 700 701 702 703 704 705 706 707
#ifdef CONFIG_NET_POLL_CONTROLLER
static void virtnet_netpoll(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_schedule(&vi->napi);
}
#endif

R
Rusty Russell 已提交
708 709 710 711
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

712
	virtnet_napi_enable(vi);
R
Rusty Russell 已提交
713 714 715
	return 0;
}

716 717 718 719 720 721 722 723
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
 * never fail unless improperly formated.
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
				 struct scatterlist *data, int out, int in)
{
724
	struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
725 726 727
	struct virtio_net_ctrl_hdr ctrl;
	virtio_net_ctrl_ack status = ~0;
	unsigned int tmp;
728
	int i;
729

730 731 732
	/* Caller should know better */
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
		(out + in > VIRTNET_SEND_COMMAND_SG_MAX));
733 734 735 736 737 738 739 740 741 742

	out++; /* Add header */
	in++; /* Add return status */

	ctrl.class = class;
	ctrl.cmd = cmd;

	sg_init_table(sg, out + in);

	sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
743 744
	for_each_sg(data, s, out + in - 2, i)
		sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
745 746
	sg_set_buf(&sg[out + in - 1], &status, sizeof(status));

747
	BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi) < 0);
748

749
	virtqueue_kick(vi->cvq);
750 751 752 753 754

	/*
	 * Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
755
	while (!virtqueue_get_buf(vi->cvq, &tmp))
756 757 758 759 760
		cpu_relax();

	return status == VIRTIO_NET_OK;
}

R
Rusty Russell 已提交
761 762 763 764 765 766 767 768 769
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);

	napi_disable(&vi->napi);

	return 0;
}

770 771 772
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
773
	struct scatterlist sg[2];
774
	u8 promisc, allmulti;
775
	struct virtio_net_ctrl_mac *mac_data;
J
Jiri Pirko 已提交
776
	struct netdev_hw_addr *ha;
777
	int uc_count;
778
	int mc_count;
779 780
	void *buf;
	int i;
781 782 783 784 785

	/* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

786 787
	promisc = ((dev->flags & IFF_PROMISC) != 0);
	allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
788

789
	sg_init_one(sg, &promisc, sizeof(promisc));
790 791 792

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_PROMISC,
793
				  sg, 1, 0))
794 795 796
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
			 promisc ? "en" : "dis");

797
	sg_init_one(sg, &allmulti, sizeof(allmulti));
798 799 800

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
				  VIRTIO_NET_CTRL_RX_ALLMULTI,
801
				  sg, 1, 0))
802 803
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
			 allmulti ? "en" : "dis");
804

805
	uc_count = netdev_uc_count(dev);
806
	mc_count = netdev_mc_count(dev);
807
	/* MAC filter - use one buffer for both lists */
808 809 810
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
811 812 813 814 815
	if (!buf) {
		dev_warn(&dev->dev, "No memory for MAC address buffer\n");
		return;
	}

816 817
	sg_init_table(sg, 2);

818
	/* Store the unicast list and count in the front of the buffer */
819
	mac_data->entries = uc_count;
J
Jiri Pirko 已提交
820
	i = 0;
821
	netdev_for_each_uc_addr(ha, dev)
J
Jiri Pirko 已提交
822
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
823 824

	sg_set_buf(&sg[0], mac_data,
825
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
826 827

	/* multicast list and count fill the end */
828
	mac_data = (void *)&mac_data->macs[uc_count][0];
829

830
	mac_data->entries = mc_count;
831
	i = 0;
832 833
	netdev_for_each_mc_addr(ha, dev)
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
834 835

	sg_set_buf(&sg[1], mac_data,
836
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
837 838 839 840 841 842 843

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
				  VIRTIO_NET_CTRL_MAC_TABLE_SET,
				  sg, 2, 0))
		dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");

	kfree(buf);
844 845
}

846
static void virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
847 848 849 850
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

851
	sg_init_one(&sg, &vid, sizeof(vid));
852 853 854 855 856 857

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
}

858
static void virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
859 860 861 862
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

863
	sg_init_one(&sg, &vid, sizeof(vid));
864 865 866 867 868 869

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
}

870
static const struct ethtool_ops virtnet_ethtool_ops = {
871
	.get_link = ethtool_op_get_link,
872 873
};

M
Mark McLoughlin 已提交
874 875 876 877 878 879 880 881 882 883 884
#define MIN_MTU 68
#define MAX_MTU 65535

static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
{
	if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
		return -EINVAL;
	dev->mtu = new_mtu;
	return 0;
}

885 886 887 888 889
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
890
	.ndo_set_mac_address = virtnet_set_mac_address,
891
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
892
	.ndo_change_mtu	     = virtnet_change_mtu,
893
	.ndo_get_stats64     = virtnet_stats,
894 895
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
896 897 898 899 900
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = virtnet_netpoll,
#endif
};

901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
static void virtnet_update_status(struct virtnet_info *vi)
{
	u16 v;

	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS))
		return;

	vi->vdev->config->get(vi->vdev,
			      offsetof(struct virtio_net_config, status),
			      &v, sizeof(v));

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
		return;

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
		netif_carrier_on(vi->dev);
		netif_wake_queue(vi->dev);
	} else {
		netif_carrier_off(vi->dev);
		netif_stop_queue(vi->dev);
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

	virtnet_update_status(vi);
}

R
Rusty Russell 已提交
936 937 938 939 940
static int virtnet_probe(struct virtio_device *vdev)
{
	int err;
	struct net_device *dev;
	struct virtnet_info *vi;
941 942 943 944
	struct virtqueue *vqs[3];
	vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
	const char *names[] = { "input", "output", "control" };
	int nvqs;
R
Rusty Russell 已提交
945 946 947 948 949 950 951

	/* Allocate ourselves a network device with room for our info */
	dev = alloc_etherdev(sizeof(struct virtnet_info));
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
952
	dev->priv_flags |= IFF_UNICAST_FLT;
953
	dev->netdev_ops = &virtnet_netdev;
R
Rusty Russell 已提交
954
	dev->features = NETIF_F_HIGHDMA;
955

956
	SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
R
Rusty Russell 已提交
957 958 959
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
960
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
R
Rusty Russell 已提交
961
		/* This opens up the world of extra features. */
962 963 964 965 966 967
		dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
		if (csum)
			dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;

		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
			dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
R
Rusty Russell 已提交
968 969
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
970
		/* Individual feature bits: what can host handle? */
971 972 973 974 975 976 977 978 979 980 981 982
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
			dev->hw_features |= NETIF_F_TSO;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
			dev->hw_features |= NETIF_F_TSO6;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
			dev->hw_features |= NETIF_F_TSO_ECN;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
			dev->hw_features |= NETIF_F_UFO;

		if (gso)
			dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
		/* (!csum && gso) case will be fixed by register_netdev() */
R
Rusty Russell 已提交
983 984 985
	}

	/* Configuration may specify what MAC to use.  Otherwise random. */
986
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
987 988 989
		vdev->config->get(vdev,
				  offsetof(struct virtio_net_config, mac),
				  dev->dev_addr, dev->addr_len);
990
	} else
R
Rusty Russell 已提交
991 992 993 994
		random_ether_addr(dev->dev_addr);

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
995
	netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
R
Rusty Russell 已提交
996 997
	vi->dev = dev;
	vi->vdev = vdev;
998
	vdev->priv = vi;
999
	vi->pages = NULL;
1000 1001 1002 1003 1004
	vi->stats = alloc_percpu(struct virtnet_stats);
	err = -ENOMEM;
	if (vi->stats == NULL)
		goto free;

1005
	INIT_DELAYED_WORK(&vi->refill, refill_work);
1006 1007
	sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
	sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
R
Rusty Russell 已提交
1008

1009
	/* If we can receive ANY GSO packets, we must allocate large ones. */
1010 1011 1012
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
1013 1014
		vi->big_packets = true;

1015 1016 1017
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

1018 1019 1020 1021 1022 1023
	/* We expect two virtqueues, receive then send,
	 * and optionally control. */
	nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;

	err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
	if (err)
1024
		goto free_stats;
R
Rusty Russell 已提交
1025

1026 1027
	vi->rvq = vqs[0];
	vi->svq = vqs[1];
R
Rusty Russell 已提交
1028

1029
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
1030
		vi->cvq = vqs[2];
1031 1032 1033

		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
			dev->features |= NETIF_F_HW_VLAN_FILTER;
1034 1035
	}

R
Rusty Russell 已提交
1036 1037 1038
	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
1039
		goto free_vqs;
R
Rusty Russell 已提交
1040
	}
1041 1042

	/* Last of all, set up some receive buffers. */
1043
	try_fill_recv(vi, GFP_KERNEL);
1044 1045 1046 1047 1048 1049 1050

	/* If we didn't even get one input buffer, we're useless. */
	if (vi->num == 0) {
		err = -ENOMEM;
		goto unregister;
	}

J
Jason Wang 已提交
1051 1052 1053 1054 1055 1056 1057 1058 1059
	/* Assume link up if device can't report link status,
	   otherwise get link status from config. */
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
		netif_carrier_off(dev);
		virtnet_update_status(vi);
	} else {
		vi->status = VIRTIO_NET_S_LINK_UP;
		netif_carrier_on(dev);
	}
1060

R
Rusty Russell 已提交
1061 1062 1063
	pr_debug("virtnet: registered device %s\n", dev->name);
	return 0;

1064 1065
unregister:
	unregister_netdev(dev);
1066
	cancel_delayed_work_sync(&vi->refill);
1067 1068
free_vqs:
	vdev->config->del_vqs(vdev);
1069 1070
free_stats:
	free_percpu(vi->stats);
R
Rusty Russell 已提交
1071 1072 1073 1074 1075
free:
	free_netdev(dev);
	return err;
}

1076 1077 1078
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
S
Shirley Ma 已提交
1079
	while (1) {
1080
		buf = virtqueue_detach_unused_buf(vi->svq);
S
Shirley Ma 已提交
1081 1082 1083 1084
		if (!buf)
			break;
		dev_kfree_skb(buf);
	}
1085
	while (1) {
1086
		buf = virtqueue_detach_unused_buf(vi->rvq);
1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
		if (!buf)
			break;
		if (vi->mergeable_rx_bufs || vi->big_packets)
			give_pages(vi, buf);
		else
			dev_kfree_skb(buf);
		--vi->num;
	}
	BUG_ON(vi->num != 0);
}

1098
static void __devexit virtnet_remove(struct virtio_device *vdev)
R
Rusty Russell 已提交
1099
{
1100
	struct virtnet_info *vi = vdev->priv;
1101

R
Rusty Russell 已提交
1102 1103 1104
	/* Stop all the virtqueues. */
	vdev->config->reset(vdev);

1105

1106
	unregister_netdev(vi->dev);
1107
	cancel_delayed_work_sync(&vi->refill);
S
Shirley Ma 已提交
1108 1109

	/* Free unused buffers in both send and recv, if any. */
1110
	free_unused_bufs(vi);
1111

1112 1113
	vdev->config->del_vqs(vi->vdev);

1114 1115 1116
	while (vi->pages)
		__free_pages(get_a_page(vi, GFP_KERNEL), 0);

1117
	free_percpu(vi->stats);
1118
	free_netdev(vi->dev);
R
Rusty Russell 已提交
1119 1120 1121 1122 1123 1124 1125
}

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

1126
static unsigned int features[] = {
1127 1128
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
	VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1129
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1130
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1131
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1132
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1133
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
1134 1135
};

1136
static struct virtio_driver virtio_net_driver = {
1137 1138
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
1139 1140 1141 1142 1143
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtnet_probe,
	.remove =	__devexit_p(virtnet_remove),
1144
	.config_changed = virtnet_config_changed,
R
Rusty Russell 已提交
1145 1146 1147 1148
};

static int __init init(void)
{
1149
	return register_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1150 1151 1152 1153
}

static void __exit fini(void)
{
1154
	unregister_virtio_driver(&virtio_net_driver);
R
Rusty Russell 已提交
1155 1156 1157 1158 1159 1160 1161
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");