diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index ff6c91995c96ca25389e1249d80cb42e2402c6de..ebd08b21b23432696047487367d44f76d4fa4f02 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -30,9 +30,10 @@ #include "vhost.h" -static int experimental_zcopytx; +static int experimental_zcopytx = 1; module_param(experimental_zcopytx, int, 0444); -MODULE_PARM_DESC(experimental_zcopytx, "Enable Experimental Zero Copy TX"); +MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;" + " 1 -Enable; 0 - Disable"); /* Max number of bytes transferred before requeueing the job. * Using this limit prevents one virtqueue from starving others. */ @@ -83,6 +84,8 @@ struct vhost_net { /* Number of times zerocopy TX recently failed. * Protected by tx vq lock. */ unsigned tx_zcopy_err; + /* Flush in progress. Protected by tx vq lock. */ + bool tx_flush; }; static void vhost_net_tx_packet(struct vhost_net *net) @@ -101,7 +104,11 @@ static void vhost_net_tx_err(struct vhost_net *net) static bool vhost_net_tx_select_zcopy(struct vhost_net *net) { - return net->tx_packets / 64 >= net->tx_zcopy_err; + /* TX flush waits for outstanding DMAs to be done. + * Don't start new DMAs. + */ + return !net->tx_flush && + net->tx_packets / 64 >= net->tx_zcopy_err; } static bool vhost_sock_zcopy(struct socket *sock) @@ -235,7 +242,7 @@ static void handle_tx(struct vhost_net *net) size_t hdr_size; struct socket *sock; struct vhost_ubuf_ref *uninitialized_var(ubufs); - bool zcopy; + bool zcopy, zcopy_used; /* TODO: check that we are running from vhost_worker? */ sock = rcu_dereference_check(vq->private_data, 1); @@ -313,8 +320,11 @@ static void handle_tx(struct vhost_net *net) iov_length(vq->hdr, s), hdr_size); break; } + zcopy_used = zcopy && (len >= VHOST_GOODCOPY_LEN || + vq->upend_idx != vq->done_idx); + /* use msg_control to pass vhost zerocopy ubuf info to skb */ - if (zcopy) { + if (zcopy_used) { vq->heads[vq->upend_idx].id = head; if (!vhost_net_tx_select_zcopy(net) || len < VHOST_GOODCOPY_LEN) { @@ -342,7 +352,7 @@ static void handle_tx(struct vhost_net *net) /* TODO: Check specific error and bomb out unless ENOBUFS? */ err = sock->ops->sendmsg(NULL, sock, &msg, len); if (unlikely(err < 0)) { - if (zcopy) { + if (zcopy_used) { if (ubufs) vhost_ubuf_put(ubufs); vq->upend_idx = ((unsigned)vq->upend_idx - 1) % @@ -356,7 +366,7 @@ static void handle_tx(struct vhost_net *net) if (err != len) pr_debug("Truncated TX packet: " " len %d != %zd\n", err, len); - if (!zcopy) + if (!zcopy_used) vhost_add_used_and_signal(&net->dev, vq, head, 0); else vhost_zerocopy_signal_used(net, vq); @@ -679,6 +689,17 @@ static void vhost_net_flush(struct vhost_net *n) { vhost_net_flush_vq(n, VHOST_NET_VQ_TX); vhost_net_flush_vq(n, VHOST_NET_VQ_RX); + if (n->dev.vqs[VHOST_NET_VQ_TX].ubufs) { + mutex_lock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex); + n->tx_flush = true; + mutex_unlock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex); + /* Wait for all lower device DMAs done. */ + vhost_ubuf_put_and_wait(n->dev.vqs[VHOST_NET_VQ_TX].ubufs); + mutex_lock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex); + n->tx_flush = false; + kref_init(&n->dev.vqs[VHOST_NET_VQ_TX].ubufs->kref); + mutex_unlock(&n->dev.vqs[VHOST_NET_VQ_TX].mutex); + } } static int vhost_net_release(struct inode *inode, struct file *f) @@ -686,18 +707,10 @@ static int vhost_net_release(struct inode *inode, struct file *f) struct vhost_net *n = f->private_data; struct socket *tx_sock; struct socket *rx_sock; - int i; vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); vhost_dev_stop(&n->dev); - for (i = 0; i < n->dev.nvqs; ++i) { - /* Wait for all lower device DMAs done. */ - if (n->dev.vqs[i].ubufs) - vhost_ubuf_put_and_wait(n->dev.vqs[i].ubufs); - - vhost_zerocopy_signal_used(n, &n->dev.vqs[i]); - } vhost_dev_cleanup(&n->dev, false); if (tx_sock) fput(tx_sock->file); @@ -826,6 +839,7 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd) n->tx_packets = 0; n->tx_zcopy_err = 0; + n->tx_flush = false; } mutex_unlock(&vq->mutex); @@ -942,8 +956,11 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, return vhost_net_reset_owner(n); default: mutex_lock(&n->dev.mutex); - r = vhost_dev_ioctl(&n->dev, ioctl, arg); - vhost_net_flush(n); + r = vhost_dev_ioctl(&n->dev, ioctl, argp); + if (r == -ENOIOCTLCMD) + r = vhost_vring_ioctl(&n->dev, ioctl, argp); + else + vhost_net_flush(n); mutex_unlock(&n->dev.mutex); return r; } diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c index ef88844824391c3893641c6841884591b1bdacfe..79e7e4d45eb242b42aeb20ab14dfe73aaabfa565 100644 --- a/drivers/vhost/tcm_vhost.c +++ b/drivers/vhost/tcm_vhost.c @@ -414,14 +414,12 @@ static struct tcm_vhost_cmd *vhost_scsi_allocate_cmd( { struct tcm_vhost_cmd *tv_cmd; struct tcm_vhost_nexus *tv_nexus; - struct se_session *se_sess; tv_nexus = tv_tpg->tpg_nexus; if (!tv_nexus) { pr_err("Unable to locate active struct tcm_vhost_nexus\n"); return ERR_PTR(-EIO); } - se_sess = tv_nexus->tvn_se_sess; tv_cmd = kzalloc(sizeof(struct tcm_vhost_cmd), GFP_ATOMIC); if (!tv_cmd) { @@ -970,7 +968,10 @@ static long vhost_scsi_ioctl(struct file *f, unsigned int ioctl, return vhost_scsi_set_features(vs, features); default: mutex_lock(&vs->dev.mutex); - r = vhost_dev_ioctl(&vs->dev, ioctl, arg); + r = vhost_dev_ioctl(&vs->dev, ioctl, argp); + /* TODO: flush backend after dev ioctl. */ + if (r == -ENOIOCTLCMD) + r = vhost_vring_ioctl(&vs->dev, ioctl, argp); mutex_unlock(&vs->dev.mutex); return r; } diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 5a3d0f1eaf9490c7151dd4052295e227512cf459..34389f75fe65693a4ad5baa36715159fd5759bbb 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -607,7 +607,7 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) return 0; } -static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp) +long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp) { struct file *eventfp, *filep = NULL; bool pollstart = false, pollstop = false; @@ -802,9 +802,8 @@ static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp) } /* Caller must have device mutex */ -long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, unsigned long arg) +long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) { - void __user *argp = (void __user *)arg; struct file *eventfp, *filep = NULL; struct eventfd_ctx *ctx = NULL; u64 p; @@ -875,7 +874,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, unsigned long arg) fput(filep); break; default: - r = vhost_set_vring(d, ioctl, argp); + r = -ENOIOCTLCMD; break; } done: diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 5e19e3d5db8cda6d77a516a8d3aa27bc92efbf9a..2639c58b23ab497ace850895f3322df661a965df 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -164,7 +164,8 @@ long vhost_dev_check_owner(struct vhost_dev *); long vhost_dev_reset_owner(struct vhost_dev *); void vhost_dev_cleanup(struct vhost_dev *, bool locked); void vhost_dev_stop(struct vhost_dev *); -long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, unsigned long arg); +long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); +long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp); int vhost_vq_access_ok(struct vhost_virtqueue *vq); int vhost_log_access_ok(struct vhost_dev *); diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c index e626fa553c5ae1b2db0f044fe0c987e13e5af9ec..6d25dcd2e97a9015da4c927c36b13d3763dc5827 100644 --- a/tools/virtio/virtio_test.c +++ b/tools/virtio/virtio_test.c @@ -232,7 +232,7 @@ const struct option longopts[] = { } }; -static void help() +static void help(void) { fprintf(stderr, "Usage: virtio_test [--help]" " [--no-indirect]"