Implement the logic of filling rq with XSK buffers. Signed-off-by: Xuan Zhuo <xuanz...@linux.alibaba.com> --- drivers/net/virtio/main.c | 9 +++++- drivers/net/virtio/virtio_net.h | 2 ++ drivers/net/virtio/xsk.c | 51 ++++++++++++++++++++++++++++++++- drivers/net/virtio/xsk.h | 2 ++ 4 files changed, 62 insertions(+), 2 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c index 325d39c39792..264ab8aa5da5 100644 --- a/drivers/net/virtio/main.c +++ b/drivers/net/virtio/main.c @@ -1790,6 +1790,11 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq, int err; bool oom; + if (rq->xsk.pool) { + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp); + goto kick; + } + do { if (vi->mergeable_rx_bufs) err = add_recvbuf_mergeable(vi, rq, gfp); @@ -1798,10 +1803,11 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq, else err = add_recvbuf_small(vi, rq, gfp); - oom = err == -ENOMEM; if (err) break; } while (rq->vq->num_free); + +kick: if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) { unsigned long flags; @@ -1810,6 +1816,7 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq, u64_stats_update_end_irqrestore(&rq->stats.syncp, flags); } + oom = err == -ENOMEM; return !oom; } diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h index f8b8f4f5b8b3..eaa5da0b0b3c 100644 --- a/drivers/net/virtio/virtio_net.h +++ b/drivers/net/virtio/virtio_net.h @@ -129,6 +129,8 @@ struct virtnet_rq { /* xdp rxq used by xsk */ struct xdp_rxq_info xdp_rxq; + + struct xdp_buff **xsk_buffs; } xsk; }; diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c index 0c6a8f92ae38..c54cd08e9c77 100644 --- a/drivers/net/virtio/xsk.c +++ b/drivers/net/virtio/xsk.c @@ -14,6 +14,47 @@ static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len) sg->length = len; } +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq, + struct xsk_buff_pool *pool, gfp_t gfp) +{ + struct xdp_buff **xsk_buffs; + dma_addr_t addr; + u32 len, i; + int err = 0; + int num; + + xsk_buffs = rq->xsk.xsk_buffs; + + num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free); + if (!num) + return -ENOMEM; + + len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len; + + for (i = 0; i < num; ++i) { + /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */ + addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len; + + sg_init_table(rq->sg, 1); + sg_fill_dma(rq->sg, addr, len); + + err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp); + if (err) + goto err; + } + + return num; + +err: + if (i) + err = i; + + for (; i < num; ++i) + xsk_buff_free(xsk_buffs[i]); + + return err; +} + static int virtnet_xsk_xmit_one(struct virtnet_sq *sq, struct xsk_buff_pool *pool, struct xdp_desc *desc) @@ -210,7 +251,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev, struct virtnet_sq *sq; struct device *dma_dev; dma_addr_t hdr_dma; - int err; + int err, size; /* In big_packets mode, xdp cannot work, so there is no need to * initialize xsk of rq. @@ -246,6 +287,12 @@ static int virtnet_xsk_pool_enable(struct net_device *dev, if (!dma_dev) return -EPERM; + size = virtqueue_get_vring_size(rq->vq); + + rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL); + if (!rq->xsk.xsk_buffs) + return -ENOMEM; + hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE); if (dma_mapping_error(dma_dev, hdr_dma)) return -ENOMEM; @@ -304,6 +351,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid) dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE); + kfree(rq->xsk.xsk_buffs); + return err1 | err2; } diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h index 7ebc9bda7aee..bef41a3f954e 100644 --- a/drivers/net/virtio/xsk.h +++ b/drivers/net/virtio/xsk.h @@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp); bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool, int budget); int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag); +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq, + struct xsk_buff_pool *pool, gfp_t gfp); #endif -- 2.32.0.3.g01195cf9f