net.c 31.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/* Copyright (C) 2009 Red Hat, Inc.
 * Author: Michael S. Tsirkin <mst@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.
 *
 * virtio-net server in host kernel.
 */

#include <linux/compat.h>
#include <linux/eventfd.h>
#include <linux/vhost.h>
#include <linux/virtio_net.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
15
#include <linux/moduleparam.h>
16 17 18
#include <linux/mutex.h>
#include <linux/workqueue.h>
#include <linux/file.h>
19
#include <linux/slab.h>
20
#include <linux/vmalloc.h>
21 22 23 24 25

#include <linux/net.h>
#include <linux/if_packet.h>
#include <linux/if_arp.h>
#include <linux/if_tun.h>
A
Arnd Bergmann 已提交
26
#include <linux/if_macvlan.h>
B
Basil Gor 已提交
27
#include <linux/if_vlan.h>
28 29 30 31 32

#include <net/sock.h>

#include "vhost.h"

33
static int experimental_zcopytx = 1;
34
module_param(experimental_zcopytx, int, 0444);
35 36
MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
		                       " 1 -Enable; 0 - Disable");
37

38 39 40 41
/* Max number of bytes transferred before requeueing the job.
 * Using this limit prevents one virtqueue from starving others. */
#define VHOST_NET_WEIGHT 0x80000

42 43 44 45
/* MAX number of TX used buffers for outstanding zerocopy */
#define VHOST_MAX_PEND 128
#define VHOST_GOODCOPY_LEN 256

46 47 48 49 50
/*
 * For transmit, used buffer len is unused; we override it to track buffer
 * status internally; used for zerocopy tx only.
 */
/* Lower device DMA failed */
51
#define VHOST_DMA_FAILED_LEN	((__force __virtio32)3)
52
/* Lower device DMA done */
53
#define VHOST_DMA_DONE_LEN	((__force __virtio32)2)
54
/* Lower device DMA in progress */
55
#define VHOST_DMA_IN_PROGRESS	((__force __virtio32)1)
56
/* Buffer unused */
57
#define VHOST_DMA_CLEAR_LEN	((__force __virtio32)0)
58

59
#define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
60

61 62 63
enum {
	VHOST_NET_FEATURES = VHOST_FEATURES |
			 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
J
Jason Wang 已提交
64 65
			 (1ULL << VIRTIO_NET_F_MRG_RXBUF) |
			 (1ULL << VIRTIO_F_IOMMU_PLATFORM)
66 67
};

68 69 70 71 72 73
enum {
	VHOST_NET_VQ_RX = 0,
	VHOST_NET_VQ_TX = 1,
	VHOST_NET_VQ_MAX = 2,
};

74
struct vhost_net_ubuf_ref {
75 76 77 78 79 80
	/* refcount follows semantics similar to kref:
	 *  0: object is released
	 *  1: no outstanding ubufs
	 * >1: outstanding ubufs
	 */
	atomic_t refcount;
81 82 83 84
	wait_queue_head_t wait;
	struct vhost_virtqueue *vq;
};

85 86
struct vhost_net_virtqueue {
	struct vhost_virtqueue vq;
87 88
	size_t vhost_hlen;
	size_t sock_hlen;
89 90 91 92 93 94 95 96 97
	/* vhost zerocopy support fields below: */
	/* last used idx for outstanding DMA zerocopy buffers */
	int upend_idx;
	/* first used idx for DMA done zerocopy buffers */
	int done_idx;
	/* an array of userspace buffers info */
	struct ubuf_info *ubuf_info;
	/* Reference counting for outstanding ubufs.
	 * Protected by vq mutex. Writers must also take device mutex. */
98
	struct vhost_net_ubuf_ref *ubufs;
99 100
};

101 102
struct vhost_net {
	struct vhost_dev dev;
103
	struct vhost_net_virtqueue vqs[VHOST_NET_VQ_MAX];
104
	struct vhost_poll poll[VHOST_NET_VQ_MAX];
105 106 107 108 109 110
	/* Number of TX recently submitted.
	 * Protected by tx vq lock. */
	unsigned tx_packets;
	/* Number of times zerocopy TX recently failed.
	 * Protected by tx vq lock. */
	unsigned tx_zcopy_err;
111 112
	/* Flush in progress. Protected by tx vq lock. */
	bool tx_flush;
113 114
};

115
static unsigned vhost_net_zcopy_mask __read_mostly;
116

117
static void vhost_net_enable_zcopy(int vq)
118
{
119
	vhost_net_zcopy_mask |= 0x1 << vq;
120 121
}

122 123
static struct vhost_net_ubuf_ref *
vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy)
124
{
125
	struct vhost_net_ubuf_ref *ubufs;
126 127 128 129 130 131
	/* No zero copy backend? Nothing to count. */
	if (!zcopy)
		return NULL;
	ubufs = kmalloc(sizeof(*ubufs), GFP_KERNEL);
	if (!ubufs)
		return ERR_PTR(-ENOMEM);
132
	atomic_set(&ubufs->refcount, 1);
133 134 135 136 137
	init_waitqueue_head(&ubufs->wait);
	ubufs->vq = vq;
	return ubufs;
}

138
static int vhost_net_ubuf_put(struct vhost_net_ubuf_ref *ubufs)
139
{
140 141 142 143
	int r = atomic_sub_return(1, &ubufs->refcount);
	if (unlikely(!r))
		wake_up(&ubufs->wait);
	return r;
144 145
}

146
static void vhost_net_ubuf_put_and_wait(struct vhost_net_ubuf_ref *ubufs)
147
{
148 149
	vhost_net_ubuf_put(ubufs);
	wait_event(ubufs->wait, !atomic_read(&ubufs->refcount));
150 151 152 153 154
}

static void vhost_net_ubuf_put_wait_and_free(struct vhost_net_ubuf_ref *ubufs)
{
	vhost_net_ubuf_put_and_wait(ubufs);
155 156 157
	kfree(ubufs);
}

158 159 160 161
static void vhost_net_clear_ubuf_info(struct vhost_net *n)
{
	int i;

162 163 164
	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
		kfree(n->vqs[i].ubuf_info);
		n->vqs[i].ubuf_info = NULL;
165 166 167
	}
}

A
Asias He 已提交
168
static int vhost_net_set_ubuf_info(struct vhost_net *n)
169 170 171 172
{
	bool zcopy;
	int i;

173
	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
174
		zcopy = vhost_net_zcopy_mask & (0x1 << i);
175 176 177 178 179 180 181 182 183 184
		if (!zcopy)
			continue;
		n->vqs[i].ubuf_info = kmalloc(sizeof(*n->vqs[i].ubuf_info) *
					      UIO_MAXIOV, GFP_KERNEL);
		if  (!n->vqs[i].ubuf_info)
			goto err;
	}
	return 0;

err:
185
	vhost_net_clear_ubuf_info(n);
186 187 188
	return -ENOMEM;
}

A
Asias He 已提交
189
static void vhost_net_vq_reset(struct vhost_net *n)
190 191 192
{
	int i;

193 194
	vhost_net_clear_ubuf_info(n);

195 196 197 198
	for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
		n->vqs[i].done_idx = 0;
		n->vqs[i].upend_idx = 0;
		n->vqs[i].ubufs = NULL;
199 200
		n->vqs[i].vhost_hlen = 0;
		n->vqs[i].sock_hlen = 0;
201 202 203 204
	}

}

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
static void vhost_net_tx_packet(struct vhost_net *net)
{
	++net->tx_packets;
	if (net->tx_packets < 1024)
		return;
	net->tx_packets = 0;
	net->tx_zcopy_err = 0;
}

static void vhost_net_tx_err(struct vhost_net *net)
{
	++net->tx_zcopy_err;
}

static bool vhost_net_tx_select_zcopy(struct vhost_net *net)
{
221 222 223 224 225
	/* TX flush waits for outstanding DMAs to be done.
	 * Don't start new DMAs.
	 */
	return !net->tx_flush &&
		net->tx_packets / 64 >= net->tx_zcopy_err;
226 227
}

228 229 230 231 232 233
static bool vhost_sock_zcopy(struct socket *sock)
{
	return unlikely(experimental_zcopytx) &&
		sock_flag(sock->sk, SOCK_ZEROCOPY);
}

234 235 236 237 238
/* In case of DMA done not in order in lower device driver for some reason.
 * upend_idx is used to track end of used idx, done_idx is used to track head
 * of used idx. Once lower device DMA done contiguously, we will signal KVM
 * guest used idx.
 */
239 240
static void vhost_zerocopy_signal_used(struct vhost_net *net,
				       struct vhost_virtqueue *vq)
241
{
242 243
	struct vhost_net_virtqueue *nvq =
		container_of(vq, struct vhost_net_virtqueue, vq);
244
	int i, add;
245 246
	int j = 0;

247
	for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
248 249
		if (vq->heads[i].len == VHOST_DMA_FAILED_LEN)
			vhost_net_tx_err(net);
250 251 252 253 254 255
		if (VHOST_DMA_IS_DONE(vq->heads[i].len)) {
			vq->heads[i].len = VHOST_DMA_CLEAR_LEN;
			++j;
		} else
			break;
	}
256 257 258 259 260 261 262
	while (j) {
		add = min(UIO_MAXIOV - nvq->done_idx, j);
		vhost_add_used_and_signal_n(vq->dev, vq,
					    &vq->heads[nvq->done_idx], add);
		nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
		j -= add;
	}
263 264
}

265
static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success)
266
{
267
	struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
268
	struct vhost_virtqueue *vq = ubufs->vq;
269
	int cnt;
270

271 272
	rcu_read_lock_bh();

273 274 275
	/* set len to mark this desc buffers done DMA */
	vq->heads[ubuf->desc].len = success ?
		VHOST_DMA_DONE_LEN : VHOST_DMA_FAILED_LEN;
276
	cnt = vhost_net_ubuf_put(ubufs);
277

278 279
	/*
	 * Trigger polling thread if guest stopped submitting new buffers:
280
	 * in this case, the refcount after decrement will eventually reach 1.
281 282 283 284
	 * We also trigger polling periodically after each 16 packets
	 * (the value 16 here is more or less arbitrary, it's tuned to trigger
	 * less than 10% of times).
	 */
285
	if (cnt <= 1 || !(cnt % 16))
286
		vhost_poll_queue(&vq->poll);
287 288

	rcu_read_unlock_bh();
289 290
}

J
Jason Wang 已提交
291 292 293 294 295 296 297 298 299 300 301 302 303 304
static inline unsigned long busy_clock(void)
{
	return local_clock() >> 10;
}

static bool vhost_can_busy_poll(struct vhost_dev *dev,
				unsigned long endtime)
{
	return likely(!need_resched()) &&
	       likely(!time_after(busy_clock(), endtime)) &&
	       likely(!signal_pending(current)) &&
	       !vhost_has_work(dev);
}

305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
static void vhost_net_disable_vq(struct vhost_net *n,
				 struct vhost_virtqueue *vq)
{
	struct vhost_net_virtqueue *nvq =
		container_of(vq, struct vhost_net_virtqueue, vq);
	struct vhost_poll *poll = n->poll + (nvq - n->vqs);
	if (!vq->private_data)
		return;
	vhost_poll_stop(poll);
}

static int vhost_net_enable_vq(struct vhost_net *n,
				struct vhost_virtqueue *vq)
{
	struct vhost_net_virtqueue *nvq =
		container_of(vq, struct vhost_net_virtqueue, vq);
	struct vhost_poll *poll = n->poll + (nvq - n->vqs);
	struct socket *sock;

	sock = vq->private_data;
	if (!sock)
		return 0;

	return vhost_poll_start(poll, sock->file);
}

J
Jason Wang 已提交
331 332 333 334 335 336 337
static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
				    struct vhost_virtqueue *vq,
				    struct iovec iov[], unsigned int iov_size,
				    unsigned int *out_num, unsigned int *in_num)
{
	unsigned long uninitialized_var(endtime);
	int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
J
Jason Wang 已提交
338
				  out_num, in_num, NULL, NULL);
J
Jason Wang 已提交
339 340 341 342 343 344

	if (r == vq->num && vq->busyloop_timeout) {
		preempt_disable();
		endtime = busy_clock() + vq->busyloop_timeout;
		while (vhost_can_busy_poll(vq->dev, endtime) &&
		       vhost_vq_avail_empty(vq->dev, vq))
345
			cpu_relax();
J
Jason Wang 已提交
346 347
		preempt_enable();
		r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
J
Jason Wang 已提交
348
				      out_num, in_num, NULL, NULL);
J
Jason Wang 已提交
349 350 351 352 353
	}

	return r;
}

J
Jason Wang 已提交
354 355 356 357 358 359 360 361 362
static bool vhost_exceeds_maxpend(struct vhost_net *net)
{
	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
	struct vhost_virtqueue *vq = &nvq->vq;

	return (nvq->upend_idx + vq->num - VHOST_MAX_PEND) % UIO_MAXIOV
		== nvq->done_idx;
}

363 364 365 366
/* Expects to be always run from workqueue - which acts as
 * read-size critical section for our kind of RCU. */
static void handle_tx(struct vhost_net *net)
{
367
	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
368
	struct vhost_virtqueue *vq = &nvq->vq;
369
	unsigned out, in;
370
	int head;
371 372 373 374 375 376 377 378
	struct msghdr msg = {
		.msg_name = NULL,
		.msg_namelen = 0,
		.msg_control = NULL,
		.msg_controllen = 0,
		.msg_flags = MSG_DONTWAIT,
	};
	size_t len, total_len = 0;
J
Jason Wang 已提交
379
	int err;
380
	size_t hdr_size;
A
Arnd Bergmann 已提交
381
	struct socket *sock;
382
	struct vhost_net_ubuf_ref *uninitialized_var(ubufs);
383
	bool zcopy, zcopy_used;
A
Arnd Bergmann 已提交
384

385 386
	mutex_lock(&vq->mutex);
	sock = vq->private_data;
387
	if (!sock)
388
		goto out;
389

J
Jason Wang 已提交
390 391 392
	if (!vq_iotlb_prefetch(vq))
		goto out;

M
Michael S. Tsirkin 已提交
393
	vhost_disable_notify(&net->dev, vq);
394

395
	hdr_size = nvq->vhost_hlen;
396
	zcopy = nvq->ubufs;
397 398

	for (;;) {
399 400
		/* Release DMAs done buffers first */
		if (zcopy)
401
			vhost_zerocopy_signal_used(net, vq);
402

403 404 405
		/* If more outstanding DMAs, queue the work.
		 * Handle upend_idx wrap around
		 */
J
Jason Wang 已提交
406
		if (unlikely(vhost_exceeds_maxpend(net)))
407 408
			break;

J
Jason Wang 已提交
409 410 411
		head = vhost_net_tx_get_vq_desc(net, vq, vq->iov,
						ARRAY_SIZE(vq->iov),
						&out, &in);
412
		/* On error, stop handling until the next kick. */
413
		if (unlikely(head < 0))
414
			break;
415 416
		/* Nothing new?  Wait for eventfd to tell us they refilled. */
		if (head == vq->num) {
M
Michael S. Tsirkin 已提交
417 418
			if (unlikely(vhost_enable_notify(&net->dev, vq))) {
				vhost_disable_notify(&net->dev, vq);
419 420 421 422 423 424 425 426 427 428 429
				continue;
			}
			break;
		}
		if (in) {
			vq_err(vq, "Unexpected descriptor format for TX: "
			       "out %d, int %d\n", out, in);
			break;
		}
		/* Skip header. TODO: support TSO. */
		len = iov_length(vq->iov, out);
A
Al Viro 已提交
430
		iov_iter_init(&msg.msg_iter, WRITE, vq->iov, out, len);
431
		iov_iter_advance(&msg.msg_iter, hdr_size);
432
		/* Sanity check */
A
Al Viro 已提交
433
		if (!msg_data_left(&msg)) {
434 435
			vq_err(vq, "Unexpected header len for TX: "
			       "%zd expected %zd\n",
436
			       len, hdr_size);
437 438
			break;
		}
A
Al Viro 已提交
439
		len = msg_data_left(&msg);
440 441 442 443 444

		zcopy_used = zcopy && len >= VHOST_GOODCOPY_LEN
				   && (nvq->upend_idx + 1) % UIO_MAXIOV !=
				      nvq->done_idx
				   && vhost_net_tx_select_zcopy(net);
445

446
		/* use msg_control to pass vhost zerocopy ubuf info to skb */
447
		if (zcopy_used) {
448 449 450
			struct ubuf_info *ubuf;
			ubuf = nvq->ubuf_info + nvq->upend_idx;

451
			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
452 453 454 455 456 457 458
			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
			ubuf->callback = vhost_zerocopy_callback;
			ubuf->ctx = nvq->ubufs;
			ubuf->desc = nvq->upend_idx;
			msg.msg_control = ubuf;
			msg.msg_controllen = sizeof(ubuf);
			ubufs = nvq->ubufs;
459
			atomic_inc(&ubufs->refcount);
460
			nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
461
		} else {
462
			msg.msg_control = NULL;
463 464
			ubufs = NULL;
		}
J
Jason Wang 已提交
465 466 467 468 469 470 471 472 473 474

		total_len += len;
		if (total_len < VHOST_NET_WEIGHT &&
		    !vhost_vq_avail_empty(&net->dev, vq) &&
		    likely(!vhost_exceeds_maxpend(net))) {
			msg.msg_flags |= MSG_MORE;
		} else {
			msg.msg_flags &= ~MSG_MORE;
		}

475
		/* TODO: Check specific error and bomb out unless ENOBUFS? */
476
		err = sock->ops->sendmsg(sock, &msg, len);
477
		if (unlikely(err < 0)) {
478
			if (zcopy_used) {
479
				vhost_net_ubuf_put(ubufs);
480 481
				nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
					% UIO_MAXIOV;
482
			}
483
			vhost_discard_vq_desc(vq, 1);
484 485 486
			break;
		}
		if (err != len)
487 488
			pr_debug("Truncated TX packet: "
				 " len %d != %zd\n", err, len);
489
		if (!zcopy_used)
490
			vhost_add_used_and_signal(&net->dev, vq, head, 0);
491
		else
492 493
			vhost_zerocopy_signal_used(net, vq);
		vhost_net_tx_packet(net);
494 495 496 497 498
		if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
			vhost_poll_queue(&vq->poll);
			break;
		}
	}
499
out:
500 501 502
	mutex_unlock(&vq->mutex);
}

503 504
static int peek_head_len(struct sock *sk)
{
J
Jason Wang 已提交
505
	struct socket *sock = sk->sk_socket;
506 507
	struct sk_buff *head;
	int len = 0;
508
	unsigned long flags;
509

J
Jason Wang 已提交
510 511 512
	if (sock->ops->peek_len)
		return sock->ops->peek_len(sock);

513
	spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
514
	head = skb_peek(&sk->sk_receive_queue);
B
Basil Gor 已提交
515
	if (likely(head)) {
516
		len = head->len;
517
		if (skb_vlan_tag_present(head))
B
Basil Gor 已提交
518 519 520
			len += VLAN_HLEN;
	}

521
	spin_unlock_irqrestore(&sk->sk_receive_queue.lock, flags);
522 523 524
	return len;
}

J
Jason Wang 已提交
525 526 527 528 529 530 531 532 533 534
static int sk_has_rx_data(struct sock *sk)
{
	struct socket *sock = sk->sk_socket;

	if (sock->ops->peek_len)
		return sock->ops->peek_len(sock);

	return skb_queue_empty(&sk->sk_receive_queue);
}

J
Jason Wang 已提交
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550
static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)
{
	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
	struct vhost_virtqueue *vq = &nvq->vq;
	unsigned long uninitialized_var(endtime);
	int len = peek_head_len(sk);

	if (!len && vq->busyloop_timeout) {
		/* Both tx vq and rx socket were polled here */
		mutex_lock(&vq->mutex);
		vhost_disable_notify(&net->dev, vq);

		preempt_disable();
		endtime = busy_clock() + vq->busyloop_timeout;

		while (vhost_can_busy_poll(&net->dev, endtime) &&
J
Jason Wang 已提交
551
		       !sk_has_rx_data(sk) &&
J
Jason Wang 已提交
552
		       vhost_vq_avail_empty(&net->dev, vq))
553
			cpu_relax();
J
Jason Wang 已提交
554 555 556 557 558 559 560 561 562 563 564 565 566

		preempt_enable();

		if (vhost_enable_notify(&net->dev, vq))
			vhost_poll_queue(&vq->poll);
		mutex_unlock(&vq->mutex);

		len = peek_head_len(sk);
	}

	return len;
}

567 568 569 570 571 572 573
/* This is a multi-buffer version of vhost_get_desc, that works if
 *	vq has read descriptors only.
 * @vq		- the relevant virtqueue
 * @datalen	- data length we'll be reading
 * @iovcount	- returned count of io vectors we fill
 * @log		- vhost log
 * @log_num	- log offset
574
 * @quota       - headcount quota, 1 for big buffer
575 576 577 578 579 580 581
 *	returns number of buffer heads allocated, negative on error
 */
static int get_rx_bufs(struct vhost_virtqueue *vq,
		       struct vring_used_elem *heads,
		       int datalen,
		       unsigned *iovcount,
		       struct vhost_log *log,
582 583
		       unsigned *log_num,
		       unsigned int quota)
584 585 586 587 588 589
{
	unsigned int out, in;
	int seg = 0;
	int headcount = 0;
	unsigned d;
	int r, nlogs = 0;
590 591 592 593
	/* len is always initialized before use since we are always called with
	 * datalen > 0.
	 */
	u32 uninitialized_var(len);
594

595
	while (datalen > 0 && headcount < quota) {
J
Jason Wang 已提交
596
		if (unlikely(seg >= UIO_MAXIOV)) {
597 598 599
			r = -ENOBUFS;
			goto err;
		}
600
		r = vhost_get_vq_desc(vq, vq->iov + seg,
601 602
				      ARRAY_SIZE(vq->iov) - seg, &out,
				      &in, log, log_num);
603 604 605 606
		if (unlikely(r < 0))
			goto err;

		d = r;
607 608 609 610 611 612 613 614 615 616 617 618 619 620
		if (d == vq->num) {
			r = 0;
			goto err;
		}
		if (unlikely(out || in <= 0)) {
			vq_err(vq, "unexpected descriptor format for RX: "
				"out %d, in %d\n", out, in);
			r = -EINVAL;
			goto err;
		}
		if (unlikely(log)) {
			nlogs += *log_num;
			log += *log_num;
		}
621 622 623 624
		heads[headcount].id = cpu_to_vhost32(vq, d);
		len = iov_length(vq->iov + seg, in);
		heads[headcount].len = cpu_to_vhost32(vq, len);
		datalen -= len;
625 626 627
		++headcount;
		seg += in;
	}
628
	heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen);
629 630 631
	*iovcount = seg;
	if (unlikely(log))
		*log_num = nlogs;
632 633 634 635 636 637

	/* Detect overrun */
	if (unlikely(datalen > 0)) {
		r = UIO_MAXIOV + 1;
		goto err;
	}
638 639 640 641 642 643
	return headcount;
err:
	vhost_discard_vq_desc(vq, headcount);
	return r;
}

644 645
/* Expects to be always run from workqueue - which acts as
 * read-size critical section for our kind of RCU. */
646
static void handle_rx(struct vhost_net *net)
647
{
648 649
	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
	struct vhost_virtqueue *vq = &nvq->vq;
650 651 652 653 654 655 656 657 658
	unsigned uninitialized_var(in), log;
	struct vhost_log *vq_log;
	struct msghdr msg = {
		.msg_name = NULL,
		.msg_namelen = 0,
		.msg_control = NULL, /* FIXME: get and handle RX aux data. */
		.msg_controllen = 0,
		.msg_flags = MSG_DONTWAIT,
	};
659 660 661
	struct virtio_net_hdr hdr = {
		.flags = 0,
		.gso_type = VIRTIO_NET_HDR_GSO_NONE
662 663
	};
	size_t total_len = 0;
664 665
	int err, mergeable;
	s16 headcount;
666 667
	size_t vhost_hlen, sock_hlen;
	size_t vhost_len, sock_len;
668
	struct socket *sock;
669
	struct iov_iter fixup;
670
	__virtio16 num_buffers;
671 672

	mutex_lock(&vq->mutex);
673 674 675
	sock = vq->private_data;
	if (!sock)
		goto out;
J
Jason Wang 已提交
676 677 678 679

	if (!vq_iotlb_prefetch(vq))
		goto out;

M
Michael S. Tsirkin 已提交
680
	vhost_disable_notify(&net->dev, vq);
681
	vhost_net_disable_vq(net, vq);
682

683 684
	vhost_hlen = nvq->vhost_hlen;
	sock_hlen = nvq->sock_hlen;
685

686
	vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
687
		vq->log : NULL;
688
	mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
689

J
Jason Wang 已提交
690
	while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk))) {
691 692 693
		sock_len += sock_hlen;
		vhost_len = sock_len + vhost_hlen;
		headcount = get_rx_bufs(vq, vq->heads, vhost_len,
694 695
					&in, vq_log, &log,
					likely(mergeable) ? UIO_MAXIOV : 1);
696 697
		/* On error, stop handling until the next kick. */
		if (unlikely(headcount < 0))
698
			goto out;
699 700
		/* On overrun, truncate and discard */
		if (unlikely(headcount > UIO_MAXIOV)) {
A
Al Viro 已提交
701
			iov_iter_init(&msg.msg_iter, READ, vq->iov, 1, 1);
702
			err = sock->ops->recvmsg(sock, &msg,
703 704 705 706
						 1, MSG_DONTWAIT | MSG_TRUNC);
			pr_debug("Discarded rx packet: len %zd\n", sock_len);
			continue;
		}
707 708
		/* OK, now we need to know about added descriptors. */
		if (!headcount) {
M
Michael S. Tsirkin 已提交
709
			if (unlikely(vhost_enable_notify(&net->dev, vq))) {
710 711
				/* They have slipped one in as we were
				 * doing that: check again. */
M
Michael S. Tsirkin 已提交
712
				vhost_disable_notify(&net->dev, vq);
713 714 715 716
				continue;
			}
			/* Nothing new?  Wait for eventfd to tell us
			 * they refilled. */
717
			goto out;
718 719
		}
		/* We don't need to be notified again. */
720 721 722 723 724 725 726 727
		iov_iter_init(&msg.msg_iter, READ, vq->iov, in, vhost_len);
		fixup = msg.msg_iter;
		if (unlikely((vhost_hlen))) {
			/* We will supply the header ourselves
			 * TODO: support TSO.
			 */
			iov_iter_advance(&msg.msg_iter, vhost_hlen);
		}
728
		err = sock->ops->recvmsg(sock, &msg,
729 730 731 732 733 734 735 736 737 738
					 sock_len, MSG_DONTWAIT | MSG_TRUNC);
		/* Userspace might have consumed the packet meanwhile:
		 * it's not supposed to do this usually, but might be hard
		 * to prevent. Discard data we got (if any) and keep going. */
		if (unlikely(err != sock_len)) {
			pr_debug("Discarded rx packet: "
				 " len %d, expected %zd\n", err, sock_len);
			vhost_discard_vq_desc(vq, headcount);
			continue;
		}
739
		/* Supply virtio_net_hdr if VHOST_NET_F_VIRTIO_NET_HDR */
740 741 742 743 744
		if (unlikely(vhost_hlen)) {
			if (copy_to_iter(&hdr, sizeof(hdr),
					 &fixup) != sizeof(hdr)) {
				vq_err(vq, "Unable to write vnet_hdr "
				       "at addr %p\n", vq->iov->iov_base);
745
				goto out;
746 747 748 749 750 751
			}
		} else {
			/* Header came from socket; we'll need to patch
			 * ->num_buffers over if VIRTIO_NET_F_MRG_RXBUF
			 */
			iov_iter_advance(&fixup, sizeof(hdr));
752 753
		}
		/* TODO: Should check and handle checksum. */
754

755
		num_buffers = cpu_to_vhost16(vq, headcount);
756
		if (likely(mergeable) &&
757 758
		    copy_to_iter(&num_buffers, sizeof num_buffers,
				 &fixup) != sizeof num_buffers) {
759 760
			vq_err(vq, "Failed num_buffers write");
			vhost_discard_vq_desc(vq, headcount);
761
			goto out;
762 763 764 765 766 767 768 769
		}
		vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
					    headcount);
		if (unlikely(vq_log))
			vhost_log_write(vq, vq_log, log, vhost_len);
		total_len += vhost_len;
		if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
			vhost_poll_queue(&vq->poll);
770
			goto out;
771 772
		}
	}
773
	vhost_net_enable_vq(net, vq);
774
out:
775 776 777
	mutex_unlock(&vq->mutex);
}

778
static void handle_tx_kick(struct vhost_work *work)
779
{
780 781 782 783
	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
						  poll.work);
	struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);

784 785 786
	handle_tx(net);
}

787
static void handle_rx_kick(struct vhost_work *work)
788
{
789 790 791 792
	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
						  poll.work);
	struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev);

793 794 795
	handle_rx(net);
}

796
static void handle_tx_net(struct vhost_work *work)
797
{
798 799
	struct vhost_net *net = container_of(work, struct vhost_net,
					     poll[VHOST_NET_VQ_TX].work);
800 801 802
	handle_tx(net);
}

803
static void handle_rx_net(struct vhost_work *work)
804
{
805 806
	struct vhost_net *net = container_of(work, struct vhost_net,
					     poll[VHOST_NET_VQ_RX].work);
807 808 809 810 811
	handle_rx(net);
}

static int vhost_net_open(struct inode *inode, struct file *f)
{
812
	struct vhost_net *n;
813
	struct vhost_dev *dev;
814
	struct vhost_virtqueue **vqs;
Z
Zhi Yong Wu 已提交
815
	int i;
816

817 818 819 820 821 822
	n = kmalloc(sizeof *n, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
	if (!n) {
		n = vmalloc(sizeof *n);
		if (!n)
			return -ENOMEM;
	}
823 824
	vqs = kmalloc(VHOST_NET_VQ_MAX * sizeof(*vqs), GFP_KERNEL);
	if (!vqs) {
825
		kvfree(n);
826 827
		return -ENOMEM;
	}
828 829

	dev = &n->dev;
830 831 832 833
	vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;
	vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;
	n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;
	n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;
834 835 836 837 838
	for (i = 0; i < VHOST_NET_VQ_MAX; i++) {
		n->vqs[i].ubufs = NULL;
		n->vqs[i].ubuf_info = NULL;
		n->vqs[i].upend_idx = 0;
		n->vqs[i].done_idx = 0;
839 840
		n->vqs[i].vhost_hlen = 0;
		n->vqs[i].sock_hlen = 0;
841
	}
Z
Zhi Yong Wu 已提交
842
	vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX);
843

844 845
	vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);
	vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);
846 847 848 849 850 851 852 853 854 855 856 857

	f->private_data = n;

	return 0;
}

static struct socket *vhost_net_stop_vq(struct vhost_net *n,
					struct vhost_virtqueue *vq)
{
	struct socket *sock;

	mutex_lock(&vq->mutex);
A
Asias He 已提交
858
	sock = vq->private_data;
859
	vhost_net_disable_vq(n, vq);
A
Asias He 已提交
860
	vq->private_data = NULL;
861 862 863 864 865 866 867
	mutex_unlock(&vq->mutex);
	return sock;
}

static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,
			   struct socket **rx_sock)
{
868 869
	*tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);
	*rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);
870 871 872 873 874
}

static void vhost_net_flush_vq(struct vhost_net *n, int index)
{
	vhost_poll_flush(n->poll + index);
875
	vhost_poll_flush(&n->vqs[index].vq.poll);
876 877 878 879 880 881
}

static void vhost_net_flush(struct vhost_net *n)
{
	vhost_net_flush_vq(n, VHOST_NET_VQ_TX);
	vhost_net_flush_vq(n, VHOST_NET_VQ_RX);
882
	if (n->vqs[VHOST_NET_VQ_TX].ubufs) {
883
		mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
884
		n->tx_flush = true;
885
		mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
886
		/* Wait for all lower device DMAs done. */
887
		vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);
888
		mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
889
		n->tx_flush = false;
890
		atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);
891
		mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);
892
	}
893 894 895 896 897 898 899 900 901 902
}

static int vhost_net_release(struct inode *inode, struct file *f)
{
	struct vhost_net *n = f->private_data;
	struct socket *tx_sock;
	struct socket *rx_sock;

	vhost_net_stop(n, &tx_sock, &rx_sock);
	vhost_net_flush(n);
903
	vhost_dev_stop(&n->dev);
904
	vhost_dev_cleanup(&n->dev, false);
905
	vhost_net_vq_reset(n);
906
	if (tx_sock)
A
Al Viro 已提交
907
		sockfd_put(tx_sock);
908
	if (rx_sock)
A
Al Viro 已提交
909
		sockfd_put(rx_sock);
910 911
	/* Make sure no callbacks are outstanding */
	synchronize_rcu_bh();
912 913 914
	/* We do an extra flush before freeing memory,
	 * since jobs can re-queue themselves. */
	vhost_net_flush(n);
915
	kfree(n->dev.vqs);
916
	kvfree(n);
917 918 919 920 921 922 923 924 925 926 927
	return 0;
}

static struct socket *get_raw_socket(int fd)
{
	struct {
		struct sockaddr_ll sa;
		char  buf[MAX_ADDR_LEN];
	} uaddr;
	int uaddr_len = sizeof uaddr, r;
	struct socket *sock = sockfd_lookup(fd, &r);
K
Krishna Kumar 已提交
928

929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948
	if (!sock)
		return ERR_PTR(-ENOTSOCK);

	/* Parameter checking */
	if (sock->sk->sk_type != SOCK_RAW) {
		r = -ESOCKTNOSUPPORT;
		goto err;
	}

	r = sock->ops->getname(sock, (struct sockaddr *)&uaddr.sa,
			       &uaddr_len, 0);
	if (r)
		goto err;

	if (uaddr.sa.sll_family != AF_PACKET) {
		r = -EPFNOSUPPORT;
		goto err;
	}
	return sock;
err:
A
Al Viro 已提交
949
	sockfd_put(sock);
950 951 952
	return ERR_PTR(r);
}

A
Arnd Bergmann 已提交
953
static struct socket *get_tap_socket(int fd)
954 955 956
{
	struct file *file = fget(fd);
	struct socket *sock;
K
Krishna Kumar 已提交
957

958 959 960
	if (!file)
		return ERR_PTR(-EBADF);
	sock = tun_get_socket(file);
A
Arnd Bergmann 已提交
961 962 963
	if (!IS_ERR(sock))
		return sock;
	sock = macvtap_get_socket(file);
964 965 966 967 968 969 970 971
	if (IS_ERR(sock))
		fput(file);
	return sock;
}

static struct socket *get_socket(int fd)
{
	struct socket *sock;
K
Krishna Kumar 已提交
972

973 974 975 976 977 978
	/* special case to disable backend */
	if (fd == -1)
		return NULL;
	sock = get_raw_socket(fd);
	if (!IS_ERR(sock))
		return sock;
A
Arnd Bergmann 已提交
979
	sock = get_tap_socket(fd);
980 981 982 983 984 985 986 987 988
	if (!IS_ERR(sock))
		return sock;
	return ERR_PTR(-ENOTSOCK);
}

static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)
{
	struct socket *sock, *oldsock;
	struct vhost_virtqueue *vq;
989
	struct vhost_net_virtqueue *nvq;
990
	struct vhost_net_ubuf_ref *ubufs, *oldubufs = NULL;
991 992 993 994 995 996 997 998 999 1000 1001
	int r;

	mutex_lock(&n->dev.mutex);
	r = vhost_dev_check_owner(&n->dev);
	if (r)
		goto err;

	if (index >= VHOST_NET_VQ_MAX) {
		r = -ENOBUFS;
		goto err;
	}
1002
	vq = &n->vqs[index].vq;
1003
	nvq = &n->vqs[index];
1004 1005 1006 1007 1008
	mutex_lock(&vq->mutex);

	/* Verify that ring has been setup correctly. */
	if (!vhost_vq_access_ok(vq)) {
		r = -EFAULT;
1009
		goto err_vq;
1010 1011 1012 1013
	}
	sock = get_socket(fd);
	if (IS_ERR(sock)) {
		r = PTR_ERR(sock);
1014
		goto err_vq;
1015 1016 1017
	}

	/* start polling new socket */
A
Asias He 已提交
1018
	oldsock = vq->private_data;
1019
	if (sock != oldsock) {
1020 1021
		ubufs = vhost_net_ubuf_alloc(vq,
					     sock && vhost_sock_zcopy(sock));
1022 1023 1024 1025
		if (IS_ERR(ubufs)) {
			r = PTR_ERR(ubufs);
			goto err_ubufs;
		}
1026

K
Krishna Kumar 已提交
1027
		vhost_net_disable_vq(n, vq);
A
Asias He 已提交
1028
		vq->private_data = sock;
G
Greg Kurz 已提交
1029
		r = vhost_vq_init_access(vq);
1030
		if (r)
1031
			goto err_used;
1032 1033 1034
		r = vhost_net_enable_vq(n, vq);
		if (r)
			goto err_used;
1035

1036 1037
		oldubufs = nvq->ubufs;
		nvq->ubufs = ubufs;
1038 1039 1040

		n->tx_packets = 0;
		n->tx_zcopy_err = 0;
1041
		n->tx_flush = false;
J
Jeff Dike 已提交
1042
	}
1043

1044 1045
	mutex_unlock(&vq->mutex);

1046
	if (oldubufs) {
1047
		vhost_net_ubuf_put_wait_and_free(oldubufs);
1048
		mutex_lock(&vq->mutex);
1049
		vhost_zerocopy_signal_used(n, vq);
1050 1051
		mutex_unlock(&vq->mutex);
	}
1052

1053 1054
	if (oldsock) {
		vhost_net_flush_vq(n, index);
A
Al Viro 已提交
1055
		sockfd_put(oldsock);
1056
	}
1057

1058 1059 1060
	mutex_unlock(&n->dev.mutex);
	return 0;

1061
err_used:
A
Asias He 已提交
1062
	vq->private_data = oldsock;
1063 1064
	vhost_net_enable_vq(n, vq);
	if (ubufs)
1065
		vhost_net_ubuf_put_wait_and_free(ubufs);
1066
err_ubufs:
A
Al Viro 已提交
1067
	sockfd_put(sock);
1068 1069
err_vq:
	mutex_unlock(&vq->mutex);
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
err:
	mutex_unlock(&n->dev.mutex);
	return r;
}

static long vhost_net_reset_owner(struct vhost_net *n)
{
	struct socket *tx_sock = NULL;
	struct socket *rx_sock = NULL;
	long err;
1080
	struct vhost_umem *umem;
K
Krishna Kumar 已提交
1081

1082 1083 1084 1085
	mutex_lock(&n->dev.mutex);
	err = vhost_dev_check_owner(&n->dev);
	if (err)
		goto done;
1086 1087
	umem = vhost_dev_reset_owner_prepare();
	if (!umem) {
1088 1089 1090
		err = -ENOMEM;
		goto done;
	}
1091 1092
	vhost_net_stop(n, &tx_sock, &rx_sock);
	vhost_net_flush(n);
1093
	vhost_dev_reset_owner(&n->dev, umem);
1094
	vhost_net_vq_reset(n);
1095 1096 1097
done:
	mutex_unlock(&n->dev.mutex);
	if (tx_sock)
A
Al Viro 已提交
1098
		sockfd_put(tx_sock);
1099
	if (rx_sock)
A
Al Viro 已提交
1100
		sockfd_put(rx_sock);
1101 1102 1103 1104 1105
	return err;
}

static int vhost_net_set_features(struct vhost_net *n, u64 features)
{
1106
	size_t vhost_hlen, sock_hlen, hdr_len;
1107
	int i;
1108

1109 1110
	hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
			       (1ULL << VIRTIO_F_VERSION_1))) ?
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
			sizeof(struct virtio_net_hdr_mrg_rxbuf) :
			sizeof(struct virtio_net_hdr);
	if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
		/* vhost provides vnet_hdr */
		vhost_hlen = hdr_len;
		sock_hlen = 0;
	} else {
		/* socket provides vnet_hdr */
		vhost_hlen = 0;
		sock_hlen = hdr_len;
	}
1122 1123
	mutex_lock(&n->dev.mutex);
	if ((features & (1 << VHOST_F_LOG_ALL)) &&
J
Jason Wang 已提交
1124 1125 1126 1127 1128 1129
	    !vhost_log_access_ok(&n->dev))
		goto out_unlock;

	if ((features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) {
		if (vhost_init_device_iotlb(&n->dev, true))
			goto out_unlock;
1130
	}
J
Jason Wang 已提交
1131

1132
	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
1133
		mutex_lock(&n->vqs[i].vq.mutex);
1134
		n->vqs[i].vq.acked_features = features;
1135 1136
		n->vqs[i].vhost_hlen = vhost_hlen;
		n->vqs[i].sock_hlen = sock_hlen;
1137
		mutex_unlock(&n->vqs[i].vq.mutex);
1138 1139 1140
	}
	mutex_unlock(&n->dev.mutex);
	return 0;
J
Jason Wang 已提交
1141 1142 1143 1144

out_unlock:
	mutex_unlock(&n->dev.mutex);
	return -EFAULT;
1145 1146
}

1147 1148 1149 1150 1151
static long vhost_net_set_owner(struct vhost_net *n)
{
	int r;

	mutex_lock(&n->dev.mutex);
1152 1153 1154 1155
	if (vhost_dev_has_owner(&n->dev)) {
		r = -EBUSY;
		goto out;
	}
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
	r = vhost_net_set_ubuf_info(n);
	if (r)
		goto out;
	r = vhost_dev_set_owner(&n->dev);
	if (r)
		vhost_net_clear_ubuf_info(n);
	vhost_net_flush(n);
out:
	mutex_unlock(&n->dev.mutex);
	return r;
}

1168 1169 1170 1171 1172 1173 1174 1175 1176
static long vhost_net_ioctl(struct file *f, unsigned int ioctl,
			    unsigned long arg)
{
	struct vhost_net *n = f->private_data;
	void __user *argp = (void __user *)arg;
	u64 __user *featurep = argp;
	struct vhost_vring_file backend;
	u64 features;
	int r;
K
Krishna Kumar 已提交
1177

1178 1179
	switch (ioctl) {
	case VHOST_NET_SET_BACKEND:
1180 1181
		if (copy_from_user(&backend, argp, sizeof backend))
			return -EFAULT;
1182 1183
		return vhost_net_set_backend(n, backend.index, backend.fd);
	case VHOST_GET_FEATURES:
1184
		features = VHOST_NET_FEATURES;
1185 1186 1187
		if (copy_to_user(featurep, &features, sizeof features))
			return -EFAULT;
		return 0;
1188
	case VHOST_SET_FEATURES:
1189 1190
		if (copy_from_user(&features, featurep, sizeof features))
			return -EFAULT;
1191
		if (features & ~VHOST_NET_FEATURES)
1192 1193 1194 1195
			return -EOPNOTSUPP;
		return vhost_net_set_features(n, features);
	case VHOST_RESET_OWNER:
		return vhost_net_reset_owner(n);
1196 1197
	case VHOST_SET_OWNER:
		return vhost_net_set_owner(n);
1198 1199
	default:
		mutex_lock(&n->dev.mutex);
1200 1201 1202 1203 1204
		r = vhost_dev_ioctl(&n->dev, ioctl, argp);
		if (r == -ENOIOCTLCMD)
			r = vhost_vring_ioctl(&n->dev, ioctl, argp);
		else
			vhost_net_flush(n);
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217
		mutex_unlock(&n->dev.mutex);
		return r;
	}
}

#ifdef CONFIG_COMPAT
static long vhost_net_compat_ioctl(struct file *f, unsigned int ioctl,
				   unsigned long arg)
{
	return vhost_net_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
}
#endif

J
Jason Wang 已提交
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
static ssize_t vhost_net_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
	struct file *file = iocb->ki_filp;
	struct vhost_net *n = file->private_data;
	struct vhost_dev *dev = &n->dev;
	int noblock = file->f_flags & O_NONBLOCK;

	return vhost_chr_read_iter(dev, to, noblock);
}

static ssize_t vhost_net_chr_write_iter(struct kiocb *iocb,
					struct iov_iter *from)
{
	struct file *file = iocb->ki_filp;
	struct vhost_net *n = file->private_data;
	struct vhost_dev *dev = &n->dev;

	return vhost_chr_write_iter(dev, from);
}

static unsigned int vhost_net_chr_poll(struct file *file, poll_table *wait)
{
	struct vhost_net *n = file->private_data;
	struct vhost_dev *dev = &n->dev;

	return vhost_chr_poll(file, dev, wait);
}

1246
static const struct file_operations vhost_net_fops = {
1247 1248
	.owner          = THIS_MODULE,
	.release        = vhost_net_release,
J
Jason Wang 已提交
1249 1250 1251
	.read_iter      = vhost_net_chr_read_iter,
	.write_iter     = vhost_net_chr_write_iter,
	.poll           = vhost_net_chr_poll,
1252 1253 1254 1255 1256
	.unlocked_ioctl = vhost_net_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl   = vhost_net_compat_ioctl,
#endif
	.open           = vhost_net_open,
1257
	.llseek		= noop_llseek,
1258 1259 1260
};

static struct miscdevice vhost_net_misc = {
1261 1262 1263
	.minor = VHOST_NET_MINOR,
	.name = "vhost-net",
	.fops = &vhost_net_fops,
1264 1265
};

C
Christoph Hellwig 已提交
1266
static int vhost_net_init(void)
1267
{
1268
	if (experimental_zcopytx)
1269
		vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
1270
	return misc_register(&vhost_net_misc);
1271 1272 1273
}
module_init(vhost_net_init);

C
Christoph Hellwig 已提交
1274
static void vhost_net_exit(void)
1275 1276 1277 1278 1279 1280 1281 1282 1283
{
	misc_deregister(&vhost_net_misc);
}
module_exit(vhost_net_exit);

MODULE_VERSION("0.0.1");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Michael S. Tsirkin");
MODULE_DESCRIPTION("Host kernel accelerator for virtio net");
1284 1285
MODULE_ALIAS_MISCDEV(VHOST_NET_MINOR);
MODULE_ALIAS("devname:vhost-net");