Jason Wang <[email protected]> writes:
> Add basic support for rx busy polling.
>
> 1 byte netperf tcp_rr on mlx4 shows 116% improvement: the transaction
> rate was increased from 9151.94 to 19787.37.
>
> Cc: Rusty Russell <[email protected]>
> Cc: Michael S. Tsirkin <[email protected]>
> Cc: Vlad Yasevich <[email protected]>
> Signed-off-by: Jason Wang <[email protected]>
Nice! But I'm deeply unqualified to review it :(
Thanks,
Rusty.
> ---
> drivers/net/virtio_net.c | 241
> +++++++++++++++++++++++++++++++++++++++++++----
> 1 file changed, 225 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index 7d9f84a..a5ce604 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -27,6 +27,7 @@
> #include <linux/slab.h>
> #include <linux/cpu.h>
> #include <linux/average.h>
> +#include <net/busy_poll.h>
>
> static int napi_weight = NAPI_POLL_WEIGHT;
> module_param(napi_weight, int, 0444);
> @@ -94,8 +95,144 @@ struct receive_queue {
>
> /* Name of this receive queue: input.$index */
> char name[40];
> +
> +#ifdef CONFIG_NET_RX_BUSY_POLL
> + unsigned int state;
> +#define VIRTNET_RQ_STATE_IDLE 0
> +#define VIRTNET_RQ_STATE_NAPI 1 /* NAPI or refill owns this
> RQ */
> +#define VIRTNET_RQ_STATE_POLL 2 /* poll owns this RQ */
> +#define VIRTNET_RQ_STATE_DISABLED 4 /* RQ is disabled */
> +#define VIRTNET_RQ_OWNED (VIRTNET_RQ_STATE_NAPI | VIRTNET_RQ_STATE_POLL)
> +#define VIRTNET_RQ_LOCKED (VIRTNET_RQ_OWNED | VIRTNET_RQ_STATE_DISABLED)
> +#define VIRTNET_RQ_STATE_NAPI_YIELD 8 /* NAPI or refill yielded this RQ
> */
> +#define VIRTNET_RQ_STATE_POLL_YIELD 16 /* poll yielded this RQ */
> +#define VIRTNET_RQ_YIELD (VIRTNET_RQ_STATE_NAPI_YIELD |
> VIRTNET_RQ_STATE_POLL_YIELD)
> + spinlock_t lock;
> +#endif /* CONFIG_NET_RX_BUSY_POLL */
> };
>
> +#ifdef CONFIG_NET_RX_BUSY_POLL
> +static inline void virtnet_rq_init_lock(struct receive_queue *rq)
> +{
> +
> + spin_lock_init(&rq->lock);
> + rq->state = VIRTNET_RQ_STATE_IDLE;
> +}
> +
> +/* called from the device poll routine or refill routine to get ownership of
> a
> + * receive queue.
> + */
> +static inline bool virtnet_rq_lock_napi_refill(struct receive_queue *rq)
> +{
> + int rc = true;
> +
> + spin_lock(&rq->lock);
> + if (rq->state & VIRTNET_RQ_LOCKED) {
> + WARN_ON(rq->state & VIRTNET_RQ_STATE_NAPI);
> + rq->state |= VIRTNET_RQ_STATE_NAPI_YIELD;
> + rc = false;
> + } else
> + /* we don't care if someone yielded */
> + rq->state = VIRTNET_RQ_STATE_NAPI;
> + spin_unlock(&rq->lock);
> + return rc;
> +}
> +
> +/* returns true is someone tried to get the rq while napi or refill had it */
> +static inline bool virtnet_rq_unlock_napi_refill(struct receive_queue *rq)
> +{
> + int rc = false;
> +
> + spin_lock(&rq->lock);
> + WARN_ON(rq->state & (VIRTNET_RQ_STATE_POLL |
> + VIRTNET_RQ_STATE_NAPI_YIELD));
> +
> + if (rq->state & VIRTNET_RQ_STATE_POLL_YIELD)
> + rc = true;
> + /* will reset state to idle, unless RQ is disabled */
> + rq->state &= VIRTNET_RQ_STATE_DISABLED;
> + spin_unlock(&rq->lock);
> + return rc;
> +}
> +
> +/* called from virtnet_low_latency_recv() */
> +static inline bool virtnet_rq_lock_poll(struct receive_queue *rq)
> +{
> + int rc = true;
> +
> + spin_lock_bh(&rq->lock);
> + if ((rq->state & VIRTNET_RQ_LOCKED)) {
> + rq->state |= VIRTNET_RQ_STATE_POLL_YIELD;
> + rc = false;
> + } else
> + /* preserve yield marks */
> + rq->state |= VIRTNET_RQ_STATE_POLL;
> + spin_unlock_bh(&rq->lock);
> + return rc;
> +}
> +
> +/* returns true if someone tried to get the receive queue while it was
> locked */
> +static inline bool virtnet_rq_unlock_poll(struct receive_queue *rq)
> +{
> + int rc = false;
> +
> + spin_lock_bh(&rq->lock);
> + WARN_ON(rq->state & (VIRTNET_RQ_STATE_NAPI));
> +
> + if (rq->state & VIRTNET_RQ_STATE_POLL_YIELD)
> + rc = true;
> + /* will reset state to idle, unless RQ is disabled */
> + rq->state &= VIRTNET_RQ_STATE_DISABLED;
> + spin_unlock_bh(&rq->lock);
> + return rc;
> +}
> +
> +/* return false if RQ is currently owned */
> +static inline bool virtnet_rq_disable(struct receive_queue *rq)
> +{
> + int rc = true;
> +
> + spin_lock_bh(&rq->lock);
> + if (rq->state & VIRTNET_RQ_OWNED)
> + rc = false;
> + rq->state |= VIRTNET_RQ_STATE_DISABLED;
> + spin_unlock_bh(&rq->lock);
> +
> + return rc;
> +}
> +
> +#else /* CONFIG_NET_RX_BUSY_POLL */
> +static inline void virtnet_rq_init_lock(struct receive_queue *rq)
> +{
> +}
> +
> +static inline bool virtnet_rq_lock_napi_refill(struct receive_queue *rq)
> +{
> + return true;
> +}
> +
> +static inline bool virtnet_rq_unlock_napi_refill(struct receive_queue *rq)
> +{
> + return false;
> +}
> +
> +static inline bool virtnet_rq_lock_poll(struct receive_queue *rq)
> +{
> + return false;
> +}
> +
> +static inline bool virtnet_rq_unlock_poll(struct receive_queue *rq)
> +{
> + return false;
> +}
> +
> +static inline bool virtnet_rq_disable(struct receive_queue *rq)
> +{
> + return true;
> +}
> +
> +#endif /* CONFIG_NET_RX_BUSY_POLL */
> +
> struct virtnet_info {
> struct virtio_device *vdev;
> struct virtqueue *cvq;
> @@ -521,6 +658,8 @@ static void receive_buf(struct receive_queue *rq, void
> *buf, unsigned int len)
> skb_shinfo(skb)->gso_segs = 0;
> }
>
> + skb_mark_napi_id(skb, &rq->napi);
> +
> netif_receive_skb(skb);
> return;
>
> @@ -714,7 +853,12 @@ static void refill_work(struct work_struct *work)
> struct receive_queue *rq = &vi->rq[i];
>
> napi_disable(&rq->napi);
> + if (!virtnet_rq_lock_napi_refill(rq)) {
> + virtnet_napi_enable(rq);
> + continue;
> + }
> still_empty = !try_fill_recv(rq, GFP_KERNEL);
> + virtnet_rq_unlock_napi_refill(rq);
> virtnet_napi_enable(rq);
>
> /* In theory, this can happen: if we don't get any buffers in
> @@ -725,16 +869,13 @@ static void refill_work(struct work_struct *work)
> }
> }
>
> -static int virtnet_poll(struct napi_struct *napi, int budget)
> +static int virtnet_receive(struct receive_queue *rq, int budget)
> {
> - struct receive_queue *rq =
> - container_of(napi, struct receive_queue, napi);
> struct virtnet_info *vi = rq->vq->vdev->priv;
> void *buf;
> - unsigned int r, len, received = 0;
> + unsigned int len, received = 0;
>
> -again:
> - while (received < budget &&
> + while ((received < budget) &&
> (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
> receive_buf(rq, buf, len);
> received++;
> @@ -745,6 +886,24 @@ again:
> schedule_delayed_work(&vi->refill, 0);
> }
>
> + return received;
> +}
> +
> +static int virtnet_poll(struct napi_struct *napi, int budget)
> +{
> + struct receive_queue *rq =
> + container_of(napi, struct receive_queue, napi);
> + unsigned int r, received = 0;
> +
> +again:
> +
> + if (!virtnet_rq_lock_napi_refill(rq))
> + return budget;
> +
> + received = virtnet_receive(rq, budget);
> +
> + virtnet_rq_unlock_napi_refill(rq);
> +
> /* Out of packets? */
> if (received < budget) {
> r = virtqueue_enable_cb_prepare(rq->vq);
> @@ -760,6 +919,52 @@ again:
> return received;
> }
>
> +#ifdef CONFIG_NET_RX_BUSY_POLL
> +/* must be called with local_bh_disable()d */
> +static int virtnet_low_latency_recv(struct napi_struct *napi)
> +{
> + struct receive_queue *rq =
> + container_of(napi, struct receive_queue, napi);
> + struct virtnet_info *vi = rq->vq->vdev->priv;
> + int received;
> +
> + if (!(vi->status & VIRTIO_NET_S_LINK_UP))
> + return LL_FLUSH_FAILED;
> +
> + if (!virtnet_rq_lock_poll(rq))
> + return LL_FLUSH_BUSY;
> +
> + received = virtnet_receive(rq, 4);
> +
> + virtnet_rq_unlock_poll(rq);
> +
> + return received;
> +}
> +#endif /* CONFIG_NET_RX_BUSY_POLL */
> +
> +static void virtnet_napi_enable_all(struct virtnet_info *vi)
> +{
> + int i;
> +
> + for (i = 0; i < vi->max_queue_pairs; i++) {
> + virtnet_rq_init_lock(&vi->rq[i]);
> + virtnet_napi_enable(&vi->rq[i]);
> + }
> +}
> +
> +static void virtnet_napi_disable_all(struct virtnet_info *vi)
> +{
> + int i;
> +
> + for (i = 0; i < vi->max_queue_pairs; i++) {
> + napi_disable(&vi->rq[i].napi);
> + while (!virtnet_rq_disable(&vi->rq[i])) {
> + pr_info("RQ %d locked\n", i);
> + usleep_range(1000, 20000);
> + }
> + }
> +}
> +
> static int virtnet_open(struct net_device *dev)
> {
> struct virtnet_info *vi = netdev_priv(dev);
> @@ -770,9 +975,10 @@ static int virtnet_open(struct net_device *dev)
> /* Make sure we have some buffers: if oom use wq. */
> if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
> schedule_delayed_work(&vi->refill, 0);
> - virtnet_napi_enable(&vi->rq[i]);
> }
>
> + virtnet_napi_enable_all(vi);
> +
> return 0;
> }
>
> @@ -1076,13 +1282,11 @@ static int virtnet_set_queues(struct virtnet_info
> *vi, u16 queue_pairs)
> static int virtnet_close(struct net_device *dev)
> {
> struct virtnet_info *vi = netdev_priv(dev);
> - int i;
>
> /* Make sure refill_work doesn't re-enable napi! */
> cancel_delayed_work_sync(&vi->refill);
>
> - for (i = 0; i < vi->max_queue_pairs; i++)
> - napi_disable(&vi->rq[i].napi);
> + virtnet_napi_disable_all(vi);
>
> return 0;
> }
> @@ -1347,6 +1551,9 @@ static const struct net_device_ops virtnet_netdev = {
> #ifdef CONFIG_NET_POLL_CONTROLLER
> .ndo_poll_controller = virtnet_netpoll,
> #endif
> +#ifdef CONFIG_NET_RX_BUSY_POLL
> + .ndo_busy_poll = virtnet_low_latency_recv,
> +#endif
> };
>
> static void virtnet_config_changed_work(struct work_struct *work)
> @@ -1552,6 +1759,7 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
> vi->rq[i].pages = NULL;
> netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
> napi_weight);
> + napi_hash_add(&vi->rq[i].napi);
>
> sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
> ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
> @@ -1854,10 +2062,12 @@ static int virtnet_freeze(struct virtio_device *vdev)
> cancel_delayed_work_sync(&vi->refill);
>
> if (netif_running(vi->dev))
> - for (i = 0; i < vi->max_queue_pairs; i++) {
> - napi_disable(&vi->rq[i].napi);
> - netif_napi_del(&vi->rq[i].napi);
> - }
> + virtnet_napi_disable_all(vi);
> +
> + for (i = 0; i < vi->max_queue_pairs; i++) {
> + napi_hash_del(&vi->rq[i].napi);
> + netif_napi_del(&vi->rq[i].napi);
> + }
>
> remove_vq_common(vi);
>
> @@ -1880,8 +2090,7 @@ static int virtnet_restore(struct virtio_device *vdev)
> if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
> schedule_delayed_work(&vi->refill, 0);
>
> - for (i = 0; i < vi->max_queue_pairs; i++)
> - virtnet_napi_enable(&vi->rq[i]);
> + virtnet_napi_enable_all(vi);
> }
>
> netif_device_attach(vi->dev);
> --
> 1.9.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
