From: Leon Romanovsky <[email protected]> The uverbs CQ creation UAPI allows users to supply their own umem for a CQ. Update mana to support this workflow while preserving support for creating umem through the legacy interface.
Signed-off-by: Leon Romanovsky <[email protected]> --- drivers/infiniband/hw/mana/cq.c | 128 +++++++++++++++++++++++------------ drivers/infiniband/hw/mana/device.c | 1 + drivers/infiniband/hw/mana/main.c | 25 +++---- drivers/infiniband/hw/mana/mana_ib.h | 4 +- drivers/infiniband/hw/mana/qp.c | 42 ++++++++++-- drivers/infiniband/hw/mana/wq.c | 14 +++- 6 files changed, 147 insertions(+), 67 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 2dce1b677115..605122ecf9f9 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -5,8 +5,8 @@ #include "mana_ib.h" -int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, - struct uverbs_attr_bundle *attrs) +int mana_ib_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct uverbs_attr_bundle *attrs) { struct ib_udata *udata = &attrs->driver_udata; struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); @@ -17,7 +17,6 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct mana_ib_dev *mdev; bool is_rnic_cq; u32 doorbell; - u32 buf_size; int err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); @@ -26,44 +25,100 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->cq_handle = INVALID_MANA_HANDLE; is_rnic_cq = mana_ib_is_rnic(mdev); - if (udata) { - if (udata->inlen < offsetof(struct mana_ib_create_cq, flags)) - return -EINVAL; + if (udata->inlen < offsetof(struct mana_ib_create_cq, flags)) + return -EINVAL; - err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); - if (err) { - ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err); - return err; - } + err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); + if (err) { + ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err); + return err; + } - if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) || - attr->cqe > U32_MAX / COMP_ENTRY_SIZE) { - ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); - return -EINVAL; - } + if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) || + attr->cqe > U32_MAX / COMP_ENTRY_SIZE) { + ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe); + return -EINVAL; + } + + cq->cqe = attr->cqe; + if (!ibcq->umem) + ibcq->umem = ib_umem_get(ibdev, ucmd.buf_addr, + cq->cqe * COMP_ENTRY_SIZE, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(ibcq->umem)) + return PTR_ERR(ibcq->umem); + cq->queue.umem = ibcq->umem; + + err = mana_ib_create_queue(mdev, &cq->queue); + if (err) + return err; - cq->cqe = attr->cqe; - err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, - &cq->queue); + mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, + ibucontext); + doorbell = mana_ucontext->doorbell; + + if (is_rnic_cq) { + err = mana_ib_gd_create_cq(mdev, cq, doorbell); if (err) { - ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); - return err; + ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err); + goto err_destroy_queue; } - mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, - ibucontext); - doorbell = mana_ucontext->doorbell; - } else { - buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); - cq->cqe = buf_size / COMP_ENTRY_SIZE; - err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); + err = mana_ib_install_cq_cb(mdev, cq); if (err) { - ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); - return err; + ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err); + goto err_destroy_rnic_cq; } - doorbell = mdev->gdma_dev->doorbell; } + resp.cqid = cq->queue.id; + err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); + if (err) { + ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); + goto err_remove_cq_cb; + } + + spin_lock_init(&cq->cq_lock); + INIT_LIST_HEAD(&cq->list_send_qp); + INIT_LIST_HEAD(&cq->list_recv_qp); + + return 0; + +err_remove_cq_cb: + mana_ib_remove_cq_cb(mdev, cq); +err_destroy_rnic_cq: + mana_ib_gd_destroy_cq(mdev, cq); +err_destroy_queue: + mana_ib_destroy_queue(mdev, &cq->queue); + return err; +} + +int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct uverbs_attr_bundle *attrs) +{ + struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); + struct ib_device *ibdev = ibcq->device; + struct mana_ib_dev *mdev; + bool is_rnic_cq; + u32 doorbell; + u32 buf_size; + int err; + + mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); + + cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors; + cq->cq_handle = INVALID_MANA_HANDLE; + is_rnic_cq = mana_ib_is_rnic(mdev); + + buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE)); + cq->cqe = buf_size / COMP_ENTRY_SIZE; + err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue); + if (err) { + ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err); + return err; + } + doorbell = mdev->gdma_dev->doorbell; + if (is_rnic_cq) { err = mana_ib_gd_create_cq(mdev, cq, doorbell); if (err) { @@ -78,23 +133,12 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } } - if (udata) { - resp.cqid = cq->queue.id; - err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); - if (err) { - ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); - goto err_remove_cq_cb; - } - } - spin_lock_init(&cq->cq_lock); INIT_LIST_HEAD(&cq->list_send_qp); INIT_LIST_HEAD(&cq->list_recv_qp); return 0; -err_remove_cq_cb: - mana_ib_remove_cq_cb(mdev, cq); err_destroy_rnic_cq: mana_ib_gd_destroy_cq(mdev, cq); err_destroy_queue: diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index ccc2279ca63c..c5c5fe051424 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -21,6 +21,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .alloc_ucontext = mana_ib_alloc_ucontext, .create_ah = mana_ib_create_ah, .create_cq = mana_ib_create_cq, + .create_user_cq = mana_ib_create_user_cq, .create_qp = mana_ib_create_qp, .create_rwq_ind_table = mana_ib_create_rwq_ind_table, .create_wq = mana_ib_create_wq, diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index fac159f7128d..a871b8287dc9 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -261,35 +261,26 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_qu return 0; } -int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, +int mana_ib_create_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue) { - struct ib_umem *umem; int err; - queue->umem = NULL; queue->id = INVALID_QUEUE_ID; queue->gdma_region = GDMA_INVALID_DMA_REGION; - umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) { - ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem); - return PTR_ERR(umem); - } - - err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region); + err = mana_ib_create_zero_offset_dma_region(mdev, queue->umem, + &queue->gdma_region); if (err) { - ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", err); - goto free_umem; + ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", + err); + return err; } - queue->umem = umem; - ibdev_dbg(&mdev->ib_dev, "created dma region 0x%llx\n", queue->gdma_region); + ibdev_dbg(&mdev->ib_dev, "created dma region 0x%llx\n", + queue->gdma_region); return 0; -free_umem: - ib_umem_release(umem); - return err; } void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue) diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index a7c8c0fd7019..3bc7c88dc136 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -624,7 +624,7 @@ int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type, struct mana_ib_queue *queue); -int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, +int mana_ib_create_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); @@ -667,6 +667,8 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd, int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs); +int mana_ib_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct uverbs_attr_bundle *attrs); int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 48c1f4977f21..b08dbc675741 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -326,11 +326,20 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", ucmd.sq_buf_addr, ucmd.port); - err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); + qp->raw_sq.umem = ib_umem_get(&mdev->ib_dev, ucmd.sq_buf_addr, + ucmd.sq_buf_size, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(qp->raw_sq.umem)) { + err = PTR_ERR(qp->raw_sq.umem); + ibdev_dbg(&mdev->ib_dev, + "Failed to get umem for qp-raw, err %d\n", err); + goto err_free_vport; + } + + err = mana_ib_create_queue(mdev, &qp->raw_sq); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create queue for create qp-raw, err %d\n", err); - goto err_free_vport; + goto err_release_umem; } /* Create a WQ on the same port handle used by the Ethernet */ @@ -391,6 +400,10 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, err_destroy_queue: mana_ib_destroy_queue(mdev, &qp->raw_sq); + return err; + +err_release_umem: + ib_umem_release(qp->raw_sq.umem); err_free_vport: mana_ib_uncfg_vport(mdev, pd, port); @@ -553,13 +566,25 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, if (i == MANA_RC_SEND_QUEUE_FMR) { qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; + qp->rc_qp.queues[i].umem = NULL; continue; } - err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j], - &qp->rc_qp.queues[i]); + qp->rc_qp.queues[i].umem = ib_umem_get(&mdev->ib_dev, + ucmd.queue_buf[j], + ucmd.queue_size[j], + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(qp->rc_qp.queues[i].umem)) { + err = PTR_ERR(qp->rc_qp.queues[i].umem); + ibdev_err(&mdev->ib_dev, "Failed to get umem for queue %d, err %d\n", + i, err); + goto release_umems; + } + + err = mana_ib_create_queue(mdev, &qp->rc_qp.queues[i]); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err); - goto destroy_queues; + ib_umem_release(qp->rc_qp.queues[i].umem); + goto release_umems; } j++; } @@ -598,6 +623,13 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, while (i-- > 0) mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]); return err; + +release_umems: + while (i-- > 0) { + if (i != MANA_RC_SEND_QUEUE_FMR) + ib_umem_release(qp->rc_qp.queues[i].umem); + } + return err; } static void mana_add_qp_to_cqs(struct mana_ib_qp *qp) diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c index f959f4b9244f..be474aa8bdfc 100644 --- a/drivers/infiniband/hw/mana/wq.c +++ b/drivers/infiniband/hw/mana/wq.c @@ -31,11 +31,19 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr); - err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue); + wq->queue.umem = ib_umem_get(&mdev->ib_dev, ucmd.wq_buf_addr, + ucmd.wq_buf_size, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(wq->queue.umem)) { + err = PTR_ERR(wq->queue.umem); + ibdev_dbg(&mdev->ib_dev, "Failed to get umem for create wq, %d\n", err); + goto err_free_wq; + } + + err = mana_ib_create_queue(mdev, &wq->queue); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create queue for create wq, %d\n", err); - goto err_free_wq; + goto err_release_umem; } wq->wqe = init_attr->max_wr; @@ -43,6 +51,8 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, wq->rx_object = INVALID_MANA_HANDLE; return &wq->ibwq; +err_release_umem: + ib_umem_release(wq->queue.umem); err_free_wq: kfree(wq); -- 2.52.0
