Implement the insertion of available buffers in the indirect descriptor area of split shadow virtqueues.
Signed-off-by: Wafer Xie <[email protected]> --- hw/virtio/vhost-shadow-virtqueue.c | 183 +++++++++++++++++++++++++++-- 1 file changed, 175 insertions(+), 8 deletions(-) diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c index ecc3245138..94ad5c3a57 100644 --- a/hw/virtio/vhost-shadow-virtqueue.c +++ b/hw/virtio/vhost-shadow-virtqueue.c @@ -189,6 +189,126 @@ static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg, return true; } +/** + * Write descriptors to indirect descriptor table + * + * @svq: The shadow virtqueue + * @sg: Cache for hwaddr + * @iovec: The iovec from the guest + * @num: iovec length + * @addr: Descriptors' GPAs, if backed by guest memory + * @indirect_desc: The indirect descriptor table + * @start_idx: Starting index in the indirect descriptor table + * @total_descs: Total number of descriptors in the table + * @write: True if they are writeable descriptors + * + * Return true if success, false otherwise and print error. + */ +static bool vhost_svq_vring_write_indirect_descs(VhostShadowVirtqueue *svq, + hwaddr *sg, + const struct iovec *iovec, + size_t num, + const hwaddr *addr, + vring_desc_t *indirect_desc, + size_t start_idx, + size_t total_descs, + bool write) +{ + bool ok; + uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0; + + if (num == 0) { + return true; + } + + ok = vhost_svq_translate_addr(svq, sg, iovec, num, addr); + if (unlikely(!ok)) { + return false; + } + + for (size_t n = 0; n < num; n++) { + size_t idx = start_idx + n; + indirect_desc[idx].addr = cpu_to_le64(sg[n]); + indirect_desc[idx].len = cpu_to_le32(iovec[n].iov_len); + indirect_desc[idx].flags = flags; + if (idx + 1 < total_descs) { + indirect_desc[idx].flags |= cpu_to_le16(VRING_DESC_F_NEXT); + indirect_desc[idx].next = cpu_to_le16(idx + 1); + } + } + + return true; +} + +/** + * Add descriptors to SVQ vring using indirect descriptors + * + * @svq: The shadow virtqueue + * @out_sg: The out iovec from the guest + * @out_num: The out iovec length + * @out_addr: The out descriptors' GPAs + * @in_sg: The in iovec from the guest + * @in_num: The in iovec length + * @in_addr: The in descriptors' GPAs + * @sgs: Cache for hwaddr + * @total_descs: Total number of descriptors (out_num + in_num) + * @indirect_desc: Pre-allocated indirect descriptor table + * @indirect_iova: IOVA of the indirect descriptor table + * @indirect_size: Size of the indirect descriptor table + * + * Return true if success, false otherwise and print error. + */ +static bool vhost_svq_add_split_indirect(VhostShadowVirtqueue *svq, + const struct iovec *out_sg, + size_t out_num, + const hwaddr *out_addr, + const struct iovec *in_sg, + size_t in_num, + const hwaddr *in_addr, + hwaddr *sgs, size_t total_descs, + vring_desc_t *indirect_desc, + hwaddr indirect_iova, + size_t indirect_size) +{ + bool ok; + + /* Populate indirect descriptor table for out descriptors */ + ok = vhost_svq_vring_write_indirect_descs(svq, sgs, out_sg, out_num, + out_addr, indirect_desc, + 0, total_descs, false); + if (unlikely(!ok)) { + svq->indirect_ops->free(svq, indirect_desc, indirect_iova, + indirect_size, svq->indirect_ops->opaque); + return false; + } + + /* Populate indirect descriptor table for in descriptors */ + ok = vhost_svq_vring_write_indirect_descs(svq, sgs, in_sg, in_num, + in_addr, indirect_desc, + out_num, total_descs, true); + if (unlikely(!ok)) { + svq->indirect_ops->free(svq, indirect_desc, indirect_iova, + indirect_size, svq->indirect_ops->opaque); + return false; + } + + /* Add a single descriptor pointing to the indirect table */ + svq->vring.desc[svq->free_head].addr = cpu_to_le64(indirect_iova); + svq->vring.desc[svq->free_head].len = + cpu_to_le32(total_descs * sizeof(vring_desc_t)); + svq->vring.desc[svq->free_head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT); + + /* Store indirect descriptor info in desc_state */ + svq->desc_state[svq->free_head].indirect_desc = indirect_desc; + svq->desc_state[svq->free_head].indirect_iova = indirect_iova; + svq->desc_state[svq->free_head].indirect_size = indirect_size; + + /* Move free_head forward */ + svq->free_head = svq->desc_next[svq->free_head]; + + return true; +} + static bool vhost_svq_add_split(VhostShadowVirtqueue *svq, const struct iovec *out_sg, size_t out_num, const hwaddr *out_addr, @@ -199,6 +319,8 @@ static bool vhost_svq_add_split(VhostShadowVirtqueue *svq, vring_avail_t *avail = svq->vring.avail; bool ok; g_autofree hwaddr *sgs = g_new(hwaddr, MAX(out_num, in_num)); + size_t total_descs = out_num + in_num; + bool use_indirect; *head = svq->free_head; @@ -209,16 +331,61 @@ static bool vhost_svq_add_split(VhostShadowVirtqueue *svq, return false; } - ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, out_addr, - in_num > 0, false); - if (unlikely(!ok)) { - return false; + /* + * Use indirect descriptors if: + * 1. Feature is negotiated + * 2. Total descriptors > 1 + * 3. Callback is available + */ + use_indirect = virtio_vdev_has_feature(svq->vdev, + VIRTIO_RING_F_INDIRECT_DESC) && + total_descs > 1 && + svq->indirect_ops && + svq->indirect_ops->alloc; + + if (use_indirect) { + vring_desc_t *indirect_desc = NULL; + hwaddr indirect_iova = 0; + size_t indirect_size = 0; + + /* Try to allocate indirect descriptor table */ + ok = svq->indirect_ops->alloc(svq, total_descs, + &indirect_desc, &indirect_iova, + &indirect_size, + svq->indirect_ops->opaque); + if (ok) { + /* Allocation succeeded, use indirect mode */ + ok = vhost_svq_add_split_indirect(svq, out_sg, out_num, out_addr, + in_sg, in_num, in_addr, + sgs, total_descs, + indirect_desc, indirect_iova, + indirect_size); + if (unlikely(!ok)) { + /* Failed to populate indirect descriptors, free and fallback */ + svq->indirect_ops->free(svq, indirect_desc, indirect_iova, + indirect_size, + svq->indirect_ops->opaque); + return false; + } + } else { + /* Allocation failed, fallback to direct mode */ + use_indirect = false; + } } - ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, in_addr, false, - true); - if (unlikely(!ok)) { - return false; + if (!use_indirect) { + /* Use direct descriptors */ + ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, out_addr, + in_num > 0, false); + if (unlikely(!ok)) { + return false; + } + + ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, in_addr, + false, true); + if (unlikely(!ok)) { + return false; + } } /* -- 2.34.1
