On Mon, Feb 28, 2011 at 12:04:43PM +0530, Krishna Kumar wrote: > Changes for mq vhost. > > vhost_net_open is changed to allocate a vhost_net and return. > The remaining initializations are delayed till SET_OWNER. > SET_OWNER is changed so that the argument is used to determine > how many txqs to use. Unmodified qemu's will pass NULL, so > this is recognized and handled as numtxqs=1. > > The number of vhost threads is <= #txqs. Threads handle more > than one txq when #txqs is more than MAX_VHOST_THREADS (4).
It is this sharing that prevents us from just reusing multiple vhost descriptors? 4 seems a bit arbitrary - do you have an explanation on why this is a good number? > The same thread handles both RX and TX - tested with tap/bridge > so far (TBD: needs some changes in macvtap driver to support > the same). > > I had to convert space->tab in vhost_attach_cgroups* to avoid > checkpatch errors. Separate patch pls, I'll apply that right away. > > Signed-off-by: Krishna Kumar <[email protected]> > --- > drivers/vhost/net.c | 295 ++++++++++++++++++++++++++-------------- > drivers/vhost/vhost.c | 225 +++++++++++++++++++----------- > drivers/vhost/vhost.h | 39 ++++- > 3 files changed, 378 insertions(+), 181 deletions(-) > > diff -ruNp org/drivers/vhost/vhost.h new/drivers/vhost/vhost.h > --- org/drivers/vhost/vhost.h 2011-02-08 09:05:09.000000000 +0530 > +++ new/drivers/vhost/vhost.h 2011-02-28 11:48:06.000000000 +0530 > @@ -35,11 +35,11 @@ struct vhost_poll { > wait_queue_t wait; > struct vhost_work work; > unsigned long mask; > - struct vhost_dev *dev; > + struct vhost_virtqueue *vq; /* points back to vq */ > }; > > void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, > - unsigned long mask, struct vhost_dev *dev); > + unsigned long mask, struct vhost_virtqueue *vq); > void vhost_poll_start(struct vhost_poll *poll, struct file *file); > void vhost_poll_stop(struct vhost_poll *poll); > void vhost_poll_flush(struct vhost_poll *poll); > @@ -108,8 +108,14 @@ struct vhost_virtqueue { > /* Log write descriptors */ > void __user *log_base; > struct vhost_log *log; > + struct task_struct *worker; /* worker for this vq */ > + spinlock_t *work_lock; /* points to a dev->work_lock[] entry */ > + struct list_head *work_list; /* points to a dev->work_list[] entry */ > + int qnum; /* 0 for RX, 1 -> n-1 for TX */ Is this right? > }; > > +#define MAX_VHOST_THREADS 4 > + > struct vhost_dev { > /* Readers use RCU to access memory table pointer > * log base pointer and features. > @@ -122,12 +128,33 @@ struct vhost_dev { > int nvqs; > struct file *log_file; > struct eventfd_ctx *log_ctx; > - spinlock_t work_lock; > - struct list_head work_list; > - struct task_struct *worker; > + spinlock_t *work_lock[MAX_VHOST_THREADS]; > + struct list_head *work_list[MAX_VHOST_THREADS]; This looks a bit strange. Won't sticking everything in a single array of structures rather than multiple arrays be better for cache utilization? > }; > > -long vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue *vqs, int > nvqs); > +/* > + * Return maximum number of vhost threads needed to handle RX & TX. > + * Upto MAX_VHOST_THREADS are started, and threads can be shared > + * among different vq's if numtxqs > MAX_VHOST_THREADS. > + */ > +static inline int get_nvhosts(int nvqs) nvhosts -> nthreads? > +{ > + return min_t(int, nvqs / 2, MAX_VHOST_THREADS); > +} > + > +/* > + * Get index of an existing thread that will handle this txq/rxq. > + * The same thread handles both rx[index] and tx[index]. > + */ > +static inline int vhost_get_thread_index(int index, int numtxqs, int nvhosts) > +{ > + return (index % numtxqs) % nvhosts; > +} > + As the only caller passes MAX_VHOST_THREADS, just use that? > +int vhost_setup_vqs(struct vhost_dev *dev, int numtxqs); > +void vhost_free_vqs(struct vhost_dev *dev); > +long vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue *vqs, int > nvqs, > + int nvhosts); > long vhost_dev_check_owner(struct vhost_dev *); > long vhost_dev_reset_owner(struct vhost_dev *); > void vhost_dev_cleanup(struct vhost_dev *); > diff -ruNp org/drivers/vhost/net.c new/drivers/vhost/net.c > --- org/drivers/vhost/net.c 2011-02-08 09:05:09.000000000 +0530 > +++ new/drivers/vhost/net.c 2011-02-28 11:48:53.000000000 +0530 > @@ -32,12 +32,6 @@ > * Using this limit prevents one virtqueue from starving others. */ > #define VHOST_NET_WEIGHT 0x80000 > > -enum { > - VHOST_NET_VQ_RX = 0, > - VHOST_NET_VQ_TX = 1, > - VHOST_NET_VQ_MAX = 2, > -}; > - > enum vhost_net_poll_state { > VHOST_NET_POLL_DISABLED = 0, > VHOST_NET_POLL_STARTED = 1, > @@ -46,12 +40,13 @@ enum vhost_net_poll_state { > > struct vhost_net { > struct vhost_dev dev; > - struct vhost_virtqueue vqs[VHOST_NET_VQ_MAX]; > - struct vhost_poll poll[VHOST_NET_VQ_MAX]; > + struct vhost_virtqueue *vqs; > + struct vhost_poll *poll; > + struct socket **socks; > /* Tells us whether we are polling a socket for TX. > * We only do this when socket buffer fills up. > * Protected by tx vq lock. */ > - enum vhost_net_poll_state tx_poll_state; > + enum vhost_net_poll_state *tx_poll_state; another array? > }; > > /* Pop first len bytes from iovec. Return number of segments used. */ > @@ -91,28 +86,28 @@ static void copy_iovec_hdr(const struct > } > > /* Caller must have TX VQ lock */ > -static void tx_poll_stop(struct vhost_net *net) > +static void tx_poll_stop(struct vhost_net *net, int qnum) > { > - if (likely(net->tx_poll_state != VHOST_NET_POLL_STARTED)) > + if (likely(net->tx_poll_state[qnum] != VHOST_NET_POLL_STARTED)) > return; > - vhost_poll_stop(net->poll + VHOST_NET_VQ_TX); > - net->tx_poll_state = VHOST_NET_POLL_STOPPED; > + vhost_poll_stop(&net->poll[qnum]); > + net->tx_poll_state[qnum] = VHOST_NET_POLL_STOPPED; > } > > /* Caller must have TX VQ lock */ > -static void tx_poll_start(struct vhost_net *net, struct socket *sock) > +static void tx_poll_start(struct vhost_net *net, struct socket *sock, int > qnum) > { > - if (unlikely(net->tx_poll_state != VHOST_NET_POLL_STOPPED)) > + if (unlikely(net->tx_poll_state[qnum] != VHOST_NET_POLL_STOPPED)) > return; > - vhost_poll_start(net->poll + VHOST_NET_VQ_TX, sock->file); > - net->tx_poll_state = VHOST_NET_POLL_STARTED; > + vhost_poll_start(&net->poll[qnum], sock->file); > + net->tx_poll_state[qnum] = VHOST_NET_POLL_STARTED; > } > > /* Expects to be always run from workqueue - which acts as > * read-size critical section for our kind of RCU. */ > -static void handle_tx(struct vhost_net *net) > +static void handle_tx(struct vhost_virtqueue *vq) > { > - struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_TX]; > + struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); > unsigned out, in, s; > int head; > struct msghdr msg = { > @@ -136,7 +131,7 @@ static void handle_tx(struct vhost_net * > wmem = atomic_read(&sock->sk->sk_wmem_alloc); > if (wmem >= sock->sk->sk_sndbuf) { > mutex_lock(&vq->mutex); > - tx_poll_start(net, sock); > + tx_poll_start(net, sock, vq->qnum); > mutex_unlock(&vq->mutex); > return; > } > @@ -145,7 +140,7 @@ static void handle_tx(struct vhost_net * > vhost_disable_notify(vq); > > if (wmem < sock->sk->sk_sndbuf / 2) > - tx_poll_stop(net); > + tx_poll_stop(net, vq->qnum); > hdr_size = vq->vhost_hlen; > > for (;;) { > @@ -160,7 +155,7 @@ static void handle_tx(struct vhost_net * > if (head == vq->num) { > wmem = atomic_read(&sock->sk->sk_wmem_alloc); > if (wmem >= sock->sk->sk_sndbuf * 3 / 4) { > - tx_poll_start(net, sock); > + tx_poll_start(net, sock, vq->qnum); > set_bit(SOCK_ASYNC_NOSPACE, &sock->flags); > break; > } > @@ -190,7 +185,7 @@ static void handle_tx(struct vhost_net * > err = sock->ops->sendmsg(NULL, sock, &msg, len); > if (unlikely(err < 0)) { > vhost_discard_vq_desc(vq, 1); > - tx_poll_start(net, sock); > + tx_poll_start(net, sock, vq->qnum); > break; > } > if (err != len) > @@ -282,9 +277,9 @@ err: > > /* Expects to be always run from workqueue - which acts as > * read-size critical section for our kind of RCU. */ > -static void handle_rx_big(struct vhost_net *net) > +static void handle_rx_big(struct vhost_virtqueue *vq, > + struct vhost_net *net) > { > - struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; > unsigned out, in, log, s; > int head; > struct vhost_log *vq_log; > @@ -392,9 +387,9 @@ static void handle_rx_big(struct vhost_n > > /* Expects to be always run from workqueue - which acts as > * read-size critical section for our kind of RCU. */ > -static void handle_rx_mergeable(struct vhost_net *net) > +static void handle_rx_mergeable(struct vhost_virtqueue *vq, > + struct vhost_net *net) > { > - struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; > unsigned uninitialized_var(in), log; > struct vhost_log *vq_log; > struct msghdr msg = { > @@ -498,99 +493,196 @@ static void handle_rx_mergeable(struct v > mutex_unlock(&vq->mutex); > } > > -static void handle_rx(struct vhost_net *net) > +static void handle_rx(struct vhost_virtqueue *vq) > { > + struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); > + > if (vhost_has_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF)) > - handle_rx_mergeable(net); > + handle_rx_mergeable(vq, net); > else > - handle_rx_big(net); > + handle_rx_big(vq, net); > } > > static void handle_tx_kick(struct vhost_work *work) > { > struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, > poll.work); > - struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); > > - handle_tx(net); > + handle_tx(vq); > } > > static void handle_rx_kick(struct vhost_work *work) > { > struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, > poll.work); > - struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); > > - handle_rx(net); > + handle_rx(vq); > } > > static void handle_tx_net(struct vhost_work *work) > { > - struct vhost_net *net = container_of(work, struct vhost_net, > - poll[VHOST_NET_VQ_TX].work); > - handle_tx(net); > + struct vhost_virtqueue *vq = container_of(work, struct vhost_poll, > + work)->vq; > + > + handle_tx(vq); > } > > static void handle_rx_net(struct vhost_work *work) > { > - struct vhost_net *net = container_of(work, struct vhost_net, > - poll[VHOST_NET_VQ_RX].work); > - handle_rx(net); > + struct vhost_virtqueue *vq = container_of(work, struct vhost_poll, > + work)->vq; > + > + handle_rx(vq); > } > > -static int vhost_net_open(struct inode *inode, struct file *f) > +void vhost_free_vqs(struct vhost_dev *dev) > { > - struct vhost_net *n = kmalloc(sizeof *n, GFP_KERNEL); > - struct vhost_dev *dev; > - int r; > + struct vhost_net *n = container_of(dev, struct vhost_net, dev); > + int i; > > - if (!n) > - return -ENOMEM; > + if (!n->vqs) > + return; > > - dev = &n->dev; > - n->vqs[VHOST_NET_VQ_TX].handle_kick = handle_tx_kick; > - n->vqs[VHOST_NET_VQ_RX].handle_kick = handle_rx_kick; > - r = vhost_dev_init(dev, n->vqs, VHOST_NET_VQ_MAX); > - if (r < 0) { > - kfree(n); > - return r; > + /* vhost_net_open does kzalloc, so this loop will not panic */ > + for (i = 0; i < get_nvhosts(dev->nvqs); i++) { > + kfree(dev->work_list[i]); > + kfree(dev->work_lock[i]); > } > > - vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev); > - vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev); > - n->tx_poll_state = VHOST_NET_POLL_DISABLED; > + kfree(n->socks); > + kfree(n->tx_poll_state); > + kfree(n->poll); > + kfree(n->vqs); > + > + /* > + * Reset so that vhost_net_release (which gets called when > + * vhost_dev_set_owner() call fails) will notice. > + */ > + n->vqs = NULL; > +} > > - f->private_data = n; > +int vhost_setup_vqs(struct vhost_dev *dev, int numtxqs) > +{ > + struct vhost_net *n = container_of(dev, struct vhost_net, dev); > + int nvhosts; > + int i, nvqs; > + int ret = -ENOMEM; > + > + if (numtxqs < 0 || numtxqs > VIRTIO_MAX_TXQS) > + return -EINVAL; > + > + if (numtxqs == 0) { > + /* Old qemu doesn't pass arguments to set_owner, use 1 txq */ > + numtxqs = 1; > + } > + > + /* Get total number of virtqueues */ > + nvqs = numtxqs * 2; > + > + n->vqs = kmalloc(nvqs * sizeof(*n->vqs), GFP_KERNEL); > + n->poll = kmalloc(nvqs * sizeof(*n->poll), GFP_KERNEL); > + n->socks = kmalloc(nvqs * sizeof(*n->socks), GFP_KERNEL); > + n->tx_poll_state = kmalloc(nvqs * sizeof(*n->tx_poll_state), > + GFP_KERNEL); > + if (!n->vqs || !n->poll || !n->socks || !n->tx_poll_state) > + goto err; > + > + /* Get total number of vhost threads */ > + nvhosts = get_nvhosts(nvqs); > + > + for (i = 0; i < nvhosts; i++) { > + dev->work_lock[i] = kmalloc(sizeof(*dev->work_lock[i]), > + GFP_KERNEL); > + dev->work_list[i] = kmalloc(sizeof(*dev->work_list[i]), > + GFP_KERNEL); > + if (!dev->work_lock[i] || !dev->work_list[i]) > + goto err; > + if (((unsigned long) dev->work_lock[i] & (SMP_CACHE_BYTES - 1)) > + || > + ((unsigned long) dev->work_list[i] & SMP_CACHE_BYTES - 1)) > + pr_debug("Unaligned buffer @ %d - Lock: %p List: %p\n", > + i, dev->work_lock[i], dev->work_list[i]); > + } > + > + /* 'numtxqs' RX followed by 'numtxqs' TX queues */ > + for (i = 0; i < numtxqs; i++) > + n->vqs[i].handle_kick = handle_rx_kick; > + for (; i < nvqs; i++) > + n->vqs[i].handle_kick = handle_tx_kick; > + > + ret = vhost_dev_init(dev, n->vqs, nvqs, nvhosts); > + if (ret < 0) > + goto err; > + > + for (i = 0; i < numtxqs; i++) > + vhost_poll_init(&n->poll[i], handle_rx_net, POLLIN, &n->vqs[i]); > + > + for (; i < nvqs; i++) { > + vhost_poll_init(&n->poll[i], handle_tx_net, POLLOUT, > + &n->vqs[i]); > + n->tx_poll_state[i] = VHOST_NET_POLL_DISABLED; > + } > > return 0; > + > +err: > + /* Free all pointers that may have been allocated */ > + vhost_free_vqs(dev); > + > + return ret; > +} > + > +static int vhost_net_open(struct inode *inode, struct file *f) > +{ > + struct vhost_net *n = kzalloc(sizeof *n, GFP_KERNEL); > + int ret = -ENOMEM; > + > + if (n) { > + struct vhost_dev *dev = &n->dev; > + > + f->private_data = n; > + mutex_init(&dev->mutex); > + > + /* Defer all other initialization till user does SET_OWNER */ > + ret = 0; > + } > + > + return ret; > } > > static void vhost_net_disable_vq(struct vhost_net *n, > struct vhost_virtqueue *vq) > { > + int qnum = vq->qnum; > + > if (!vq->private_data) > return; > - if (vq == n->vqs + VHOST_NET_VQ_TX) { > - tx_poll_stop(n); > - n->tx_poll_state = VHOST_NET_POLL_DISABLED; > - } else > - vhost_poll_stop(n->poll + VHOST_NET_VQ_RX); > + if (qnum < n->dev.nvqs / 2) { > + /* qnum is less than half, we are RX */ > + vhost_poll_stop(&n->poll[qnum]); > + } else { /* otherwise we are TX */ > + tx_poll_stop(n, qnum); > + n->tx_poll_state[qnum] = VHOST_NET_POLL_DISABLED; > + } > } > > static void vhost_net_enable_vq(struct vhost_net *n, > struct vhost_virtqueue *vq) > { > struct socket *sock; > + int qnum = vq->qnum; > > sock = rcu_dereference_protected(vq->private_data, > lockdep_is_held(&vq->mutex)); > if (!sock) > return; > - if (vq == n->vqs + VHOST_NET_VQ_TX) { > - n->tx_poll_state = VHOST_NET_POLL_STOPPED; > - tx_poll_start(n, sock); > - } else > - vhost_poll_start(n->poll + VHOST_NET_VQ_RX, sock->file); > + if (qnum < n->dev.nvqs / 2) { > + /* qnum is less than half, we are RX */ > + vhost_poll_start(&n->poll[qnum], sock->file); > + } else { > + n->tx_poll_state[qnum] = VHOST_NET_POLL_STOPPED; > + tx_poll_start(n, sock, qnum); > + } > } > > static struct socket *vhost_net_stop_vq(struct vhost_net *n, > @@ -607,11 +699,12 @@ static struct socket *vhost_net_stop_vq( > return sock; > } > > -static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock, > - struct socket **rx_sock) > +static void vhost_net_stop(struct vhost_net *n) > { > - *tx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_TX); > - *rx_sock = vhost_net_stop_vq(n, n->vqs + VHOST_NET_VQ_RX); > + int i; > + > + for (i = n->dev.nvqs - 1; i >= 0; i--) > + n->socks[i] = vhost_net_stop_vq(n, &n->vqs[i]); > } > > static void vhost_net_flush_vq(struct vhost_net *n, int index) > @@ -622,26 +715,33 @@ static void vhost_net_flush_vq(struct vh > > static void vhost_net_flush(struct vhost_net *n) > { > - vhost_net_flush_vq(n, VHOST_NET_VQ_TX); > - vhost_net_flush_vq(n, VHOST_NET_VQ_RX); > + int i; > + > + for (i = n->dev.nvqs - 1; i >= 0; i--) > + vhost_net_flush_vq(n, i); > } > > static int vhost_net_release(struct inode *inode, struct file *f) > { > struct vhost_net *n = f->private_data; > - struct socket *tx_sock; > - struct socket *rx_sock; > + struct vhost_dev *dev = &n->dev; > + int i; > > - vhost_net_stop(n, &tx_sock, &rx_sock); > + vhost_net_stop(n); > vhost_net_flush(n); > - vhost_dev_cleanup(&n->dev); > - if (tx_sock) > - fput(tx_sock->file); > - if (rx_sock) > - fput(rx_sock->file); > + vhost_dev_cleanup(dev); > + > + for (i = n->dev.nvqs - 1; i >= 0; i--) > + if (n->socks[i]) > + fput(n->socks[i]->file); > + > /* We do an extra flush before freeing memory, > * since jobs can re-queue themselves. */ > vhost_net_flush(n); > + > + /* Free all old pointers */ > + vhost_free_vqs(dev); > + > kfree(n); > return 0; > } > @@ -719,7 +819,7 @@ static long vhost_net_set_backend(struct > if (r) > goto err; > > - if (index >= VHOST_NET_VQ_MAX) { > + if (index >= n->dev.nvqs) { > r = -ENOBUFS; > goto err; > } > @@ -741,9 +841,9 @@ static long vhost_net_set_backend(struct > oldsock = rcu_dereference_protected(vq->private_data, > lockdep_is_held(&vq->mutex)); > if (sock != oldsock) { > - vhost_net_disable_vq(n, vq); > - rcu_assign_pointer(vq->private_data, sock); > - vhost_net_enable_vq(n, vq); > + vhost_net_disable_vq(n, vq); > + rcu_assign_pointer(vq->private_data, sock); > + vhost_net_enable_vq(n, vq); > } > > mutex_unlock(&vq->mutex); > @@ -765,22 +865,25 @@ err: > > static long vhost_net_reset_owner(struct vhost_net *n) > { > - struct socket *tx_sock = NULL; > - struct socket *rx_sock = NULL; > long err; > + int i; > + > mutex_lock(&n->dev.mutex); > err = vhost_dev_check_owner(&n->dev); > - if (err) > - goto done; > - vhost_net_stop(n, &tx_sock, &rx_sock); > + if (err) { > + mutex_unlock(&n->dev.mutex); > + return err; > + } > + > + vhost_net_stop(n); > vhost_net_flush(n); > err = vhost_dev_reset_owner(&n->dev); > -done: > mutex_unlock(&n->dev.mutex); > - if (tx_sock) > - fput(tx_sock->file); > - if (rx_sock) > - fput(rx_sock->file); > + > + for (i = n->dev.nvqs - 1; i >= 0; i--) > + if (n->socks[i]) > + fput(n->socks[i]->file); > + > return err; > } > > @@ -809,7 +912,7 @@ static int vhost_net_set_features(struct > } > n->dev.acked_features = features; > smp_wmb(); > - for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { > + for (i = 0; i < n->dev.nvqs; ++i) { > mutex_lock(&n->vqs[i].mutex); > n->vqs[i].vhost_hlen = vhost_hlen; > n->vqs[i].sock_hlen = sock_hlen; > diff -ruNp org/drivers/vhost/vhost.c new/drivers/vhost/vhost.c > --- org/drivers/vhost/vhost.c 2011-01-19 20:01:29.000000000 +0530 > +++ new/drivers/vhost/vhost.c 2011-02-25 21:18:14.000000000 +0530 > @@ -70,12 +70,12 @@ static void vhost_work_init(struct vhost > > /* Init poll structure */ > void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, > - unsigned long mask, struct vhost_dev *dev) > + unsigned long mask, struct vhost_virtqueue *vq) > { > init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup); > init_poll_funcptr(&poll->table, vhost_poll_func); > poll->mask = mask; > - poll->dev = dev; > + poll->vq = vq; > > vhost_work_init(&poll->work, fn); > } > @@ -97,29 +97,30 @@ void vhost_poll_stop(struct vhost_poll * > remove_wait_queue(poll->wqh, &poll->wait); > } > > -static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work > *work, > - unsigned seq) > +static bool vhost_work_seq_done(struct vhost_virtqueue *vq, > + struct vhost_work *work, unsigned seq) > { > int left; > - spin_lock_irq(&dev->work_lock); > + spin_lock_irq(vq->work_lock); > left = seq - work->done_seq; > - spin_unlock_irq(&dev->work_lock); > + spin_unlock_irq(vq->work_lock); > return left <= 0; > } > > -static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) > +static void vhost_work_flush(struct vhost_virtqueue *vq, > + struct vhost_work *work) > { > unsigned seq; > int flushing; > > - spin_lock_irq(&dev->work_lock); > + spin_lock_irq(vq->work_lock); > seq = work->queue_seq; > work->flushing++; > - spin_unlock_irq(&dev->work_lock); > - wait_event(work->done, vhost_work_seq_done(dev, work, seq)); > - spin_lock_irq(&dev->work_lock); > + spin_unlock_irq(vq->work_lock); > + wait_event(work->done, vhost_work_seq_done(vq, work, seq)); > + spin_lock_irq(vq->work_lock); > flushing = --work->flushing; > - spin_unlock_irq(&dev->work_lock); > + spin_unlock_irq(vq->work_lock); > BUG_ON(flushing < 0); > } > > @@ -127,26 +128,26 @@ static void vhost_work_flush(struct vhos > * locks that are also used by the callback. */ > void vhost_poll_flush(struct vhost_poll *poll) > { > - vhost_work_flush(poll->dev, &poll->work); > + vhost_work_flush(poll->vq, &poll->work); > } > > -static inline void vhost_work_queue(struct vhost_dev *dev, > +static inline void vhost_work_queue(struct vhost_virtqueue *vq, > struct vhost_work *work) > { > unsigned long flags; > > - spin_lock_irqsave(&dev->work_lock, flags); > + spin_lock_irqsave(vq->work_lock, flags); > if (list_empty(&work->node)) { > - list_add_tail(&work->node, &dev->work_list); > + list_add_tail(&work->node, vq->work_list); > work->queue_seq++; > - wake_up_process(dev->worker); > + wake_up_process(vq->worker); > } > - spin_unlock_irqrestore(&dev->work_lock, flags); > + spin_unlock_irqrestore(vq->work_lock, flags); > } > > void vhost_poll_queue(struct vhost_poll *poll) > { > - vhost_work_queue(poll->dev, &poll->work); > + vhost_work_queue(poll->vq, &poll->work); > } > > static void vhost_vq_reset(struct vhost_dev *dev, > @@ -176,17 +177,17 @@ static void vhost_vq_reset(struct vhost_ > > static int vhost_worker(void *data) > { > - struct vhost_dev *dev = data; > + struct vhost_virtqueue *vq = data; > struct vhost_work *work = NULL; > unsigned uninitialized_var(seq); > > - use_mm(dev->mm); > + use_mm(vq->dev->mm); > > for (;;) { > /* mb paired w/ kthread_stop */ > set_current_state(TASK_INTERRUPTIBLE); > > - spin_lock_irq(&dev->work_lock); > + spin_lock_irq(vq->work_lock); > if (work) { > work->done_seq = seq; > if (work->flushing) > @@ -194,18 +195,18 @@ static int vhost_worker(void *data) > } > > if (kthread_should_stop()) { > - spin_unlock_irq(&dev->work_lock); > + spin_unlock_irq(vq->work_lock); > __set_current_state(TASK_RUNNING); > break; > } > - if (!list_empty(&dev->work_list)) { > - work = list_first_entry(&dev->work_list, > + if (!list_empty(vq->work_list)) { > + work = list_first_entry(vq->work_list, > struct vhost_work, node); > list_del_init(&work->node); > seq = work->queue_seq; > } else > work = NULL; > - spin_unlock_irq(&dev->work_lock); > + spin_unlock_irq(vq->work_lock); > > if (work) { > __set_current_state(TASK_RUNNING); > @@ -214,7 +215,7 @@ static int vhost_worker(void *data) > schedule(); > > } > - unuse_mm(dev->mm); > + unuse_mm(vq->dev->mm); > return 0; > } > > @@ -258,7 +259,7 @@ static void vhost_dev_free_iovecs(struct > } > > long vhost_dev_init(struct vhost_dev *dev, > - struct vhost_virtqueue *vqs, int nvqs) > + struct vhost_virtqueue *vqs, int nvqs, int nvhosts) > { > int i; > > @@ -269,20 +270,34 @@ long vhost_dev_init(struct vhost_dev *de > dev->log_file = NULL; > dev->memory = NULL; > dev->mm = NULL; > - spin_lock_init(&dev->work_lock); > - INIT_LIST_HEAD(&dev->work_list); > - dev->worker = NULL; > > for (i = 0; i < dev->nvqs; ++i) { > - dev->vqs[i].log = NULL; > - dev->vqs[i].indirect = NULL; > - dev->vqs[i].heads = NULL; > - dev->vqs[i].dev = dev; > - mutex_init(&dev->vqs[i].mutex); > + struct vhost_virtqueue *vq = &dev->vqs[i]; > + int j; > + > + if (i < nvhosts) { > + spin_lock_init(dev->work_lock[i]); > + INIT_LIST_HEAD(dev->work_list[i]); > + j = i; > + } else { > + /* Share work with another thread */ > + j = vhost_get_thread_index(i, nvqs / 2, nvhosts); > + } > + > + vq->work_lock = dev->work_lock[j]; > + vq->work_list = dev->work_list[j]; > + > + vq->worker = NULL; > + vq->qnum = i; > + vq->log = NULL; > + vq->indirect = NULL; > + vq->heads = NULL; > + vq->dev = dev; > + mutex_init(&vq->mutex); > vhost_vq_reset(dev, dev->vqs + i); > - if (dev->vqs[i].handle_kick) > - vhost_poll_init(&dev->vqs[i].poll, > - dev->vqs[i].handle_kick, POLLIN, dev); > + if (vq->handle_kick) > + vhost_poll_init(&vq->poll, > + vq->handle_kick, POLLIN, vq); > } > > return 0; > @@ -296,65 +311,124 @@ long vhost_dev_check_owner(struct vhost_ > } > > struct vhost_attach_cgroups_struct { > - struct vhost_work work; > - struct task_struct *owner; > - int ret; > + struct vhost_work work; > + struct task_struct *owner; > + int ret; > }; > > static void vhost_attach_cgroups_work(struct vhost_work *work) > { > - struct vhost_attach_cgroups_struct *s; > - s = container_of(work, struct vhost_attach_cgroups_struct, work); > - s->ret = cgroup_attach_task_all(s->owner, current); > + struct vhost_attach_cgroups_struct *s; > + s = container_of(work, struct vhost_attach_cgroups_struct, work); > + s->ret = cgroup_attach_task_all(s->owner, current); > +} > + > +static int vhost_attach_cgroups(struct vhost_virtqueue *vq) > +{ > + struct vhost_attach_cgroups_struct attach; > + attach.owner = current; > + vhost_work_init(&attach.work, vhost_attach_cgroups_work); > + vhost_work_queue(vq, &attach.work); > + vhost_work_flush(vq, &attach.work); > + return attach.ret; > +} > + > +static void __vhost_stop_workers(struct vhost_dev *dev, int nvhosts) > +{ > + int i; > + > + for (i = 0; i < nvhosts; i++) { > + WARN_ON(!list_empty(dev->vqs[i].work_list)); > + if (dev->vqs[i].worker) { > + kthread_stop(dev->vqs[i].worker); > + dev->vqs[i].worker = NULL; > + } > + } > + > + if (dev->mm) > + mmput(dev->mm); > + dev->mm = NULL; > +} > + > +static void vhost_stop_workers(struct vhost_dev *dev) > +{ > + __vhost_stop_workers(dev, get_nvhosts(dev->nvqs)); > } > > -static int vhost_attach_cgroups(struct vhost_dev *dev) > -{ > - struct vhost_attach_cgroups_struct attach; > - attach.owner = current; > - vhost_work_init(&attach.work, vhost_attach_cgroups_work); > - vhost_work_queue(dev, &attach.work); > - vhost_work_flush(dev, &attach.work); > - return attach.ret; > +static int vhost_start_workers(struct vhost_dev *dev) > +{ > + int nvhosts = get_nvhosts(dev->nvqs); > + int i, err; > + > + for (i = 0; i < dev->nvqs; ++i) { > + struct vhost_virtqueue *vq = &dev->vqs[i]; > + > + if (i < nvhosts) { > + /* Start a new thread */ > + vq->worker = kthread_create(vhost_worker, vq, > + "vhost-%d-%d", > + current->pid, i); > + if (IS_ERR(vq->worker)) { > + i--; /* no thread to clean at this index */ > + err = PTR_ERR(vq->worker); > + goto err; > + } > + > + wake_up_process(vq->worker); > + > + /* avoid contributing to loadavg */ > + err = vhost_attach_cgroups(vq); > + if (err) > + goto err; > + } else { > + /* Share work with an existing thread */ > + int j = vhost_get_thread_index(i, dev->nvqs / 2, > + nvhosts); > + > + vq->worker = dev->vqs[j].worker; > + } > + } > + return 0; > + > +err: > + __vhost_stop_workers(dev, i); > + return err; > } > > /* Caller should have device mutex */ > -static long vhost_dev_set_owner(struct vhost_dev *dev) > +static long vhost_dev_set_owner(struct vhost_dev *dev, int numtxqs) > { > - struct task_struct *worker; > int err; > /* Is there an owner already? */ > if (dev->mm) { > err = -EBUSY; > goto err_mm; > } > + > + err = vhost_setup_vqs(dev, numtxqs); > + if (err) > + goto err_mm; > + > /* No owner, become one */ > dev->mm = get_task_mm(current); > - worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid); > - if (IS_ERR(worker)) { > - err = PTR_ERR(worker); > - goto err_worker; > - } > - > - dev->worker = worker; > - wake_up_process(worker); /* avoid contributing to loadavg */ > > - err = vhost_attach_cgroups(dev); > + /* Start threads */ > + err = vhost_start_workers(dev); > if (err) > - goto err_cgroup; > + goto free_vqs; > > err = vhost_dev_alloc_iovecs(dev); > if (err) > - goto err_cgroup; > + goto clean_workers; > > return 0; > -err_cgroup: > - kthread_stop(worker); > - dev->worker = NULL; > -err_worker: > +clean_workers: > + vhost_stop_workers(dev); > +free_vqs: > if (dev->mm) > mmput(dev->mm); > dev->mm = NULL; > + vhost_free_vqs(dev); > err_mm: > return err; > } > @@ -408,14 +482,7 @@ void vhost_dev_cleanup(struct vhost_dev > kfree(rcu_dereference_protected(dev->memory, > lockdep_is_held(&dev->mutex))); > RCU_INIT_POINTER(dev->memory, NULL); > - WARN_ON(!list_empty(&dev->work_list)); > - if (dev->worker) { > - kthread_stop(dev->worker); > - dev->worker = NULL; > - } > - if (dev->mm) > - mmput(dev->mm); > - dev->mm = NULL; > + vhost_stop_workers(dev); > } > > static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz) > @@ -775,7 +842,7 @@ long vhost_dev_ioctl(struct vhost_dev *d > > /* If you are not the owner, you can become one */ > if (ioctl == VHOST_SET_OWNER) { > - r = vhost_dev_set_owner(d); > + r = vhost_dev_set_owner(d, arg); > goto done; > } > -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
