From: David Hu <[email protected]> Support the standard 1-step queue buffer creation model for QP by using `UVERBS_ATTR_CREATE_QP_BUF_UMEM` and `ib_umem_get_attr()`.
Add `irdma_setup_q_pbl_from_umem()` to allocate the backing MR and program hardware pble directly from the pinned umem. Proper umem and pble cleanup is added to `irdma_free_qp_rsrc()` upon QP destruction. We retain backward compatibility for legacy 2-step registration if the UMEM attribute is not provided. `irdma_alloc_iwmr()` assumed there is always a valid PD, which is valid for regular MR, and QP. That is not the case for CQ. This modernization fixes `irdma_alloc_iwmr()` to take in an `ibdev` to avoid a potential NULL dereference. Note for MR teardown: This patch unifies the teardown of legacy 2-step queue MR and modern 1-step dummy MR via, `refcount_dec_and_test(&iwmr->user_ring_refs)` For legacy queue MR, ref count starts at 2 (one by queue, and one by `ib_mr`), it goes to 1 during queue teardown, while deferring memory teardown to `irdma_dereg_mr`. For 1-step dummy MR, ref count starts at 1, and teardown happens cleanly here. The teardown sequence (`ib_umem_release`, `irdma_free_pble`, `kfree`) is inlined in `utils.c` here because `irdma_free_iwmr` is statically scoped in `verbs.c`. Combining was considered, and we think it belongs to a future project. Signed-off-by: David Hu <[email protected]> --- drivers/infiniband/hw/irdma/utils.c | 12 ++- drivers/infiniband/hw/irdma/verbs.c | 132 +++++++++++++++++++++++++--- 2 files changed, 127 insertions(+), 17 deletions(-) diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 290ad02ed657..1e5694a4fd91 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -1170,9 +1170,15 @@ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) kfree(iwqp->kqp.rq_wrid_mem); if (iwqp->user_mode && iwqp->iwpbl) { - struct irdma_mr *iwmr = iwqp->iwpbl->iwmr; - - refcount_dec(&iwmr->user_ring_refs); + struct irdma_pbl *iwpbl = iwqp->iwpbl; + struct irdma_mr *iwmr = iwpbl->iwmr; + + if (iwmr && refcount_dec_and_test(&iwmr->user_ring_refs)) { + ib_umem_release(iwmr->region); + if (iwpbl->pbl_allocated) + irdma_free_pble(rf->pble_rsrc, &iwpbl->pble_alloc); + kfree(iwmr); + } } } diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index dd824720671c..578c2d4b4d3c 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -18,6 +18,12 @@ static const struct uapi_definition irdma_uapi_defs[] = { {} }; +static struct irdma_pbl *irdma_setup_q_pbl_from_umem(struct irdma_device *iwdev, + struct ib_umem *region, + struct ib_pd *pd, + enum irdma_memreg_type reg_type, + struct irdma_mem_reg_req *req); + /** * irdma_query_device - get device attributes * @ibdev: device pointer from stack @@ -670,17 +676,6 @@ static int irdma_setup_umode_qp(struct irdma_ucontext *ucontext, iwqp->ctx_info.qp_compl_ctx = req->user_compl_ctx; iwqp->user_mode = 1; - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - iwqp->iwpbl = irdma_get_pbl((unsigned long)req->user_wqe_bufs, - &ucontext->qp_reg_mem_list); - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); - - if (!iwqp->iwpbl) { - ret = -ENODATA; - ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); - return ret; - } - if (!ucontext->use_raw_attrs) { /** * Maintain backward compat with older ABI which passes sq and @@ -713,6 +708,52 @@ static int irdma_setup_umode_qp(struct irdma_ucontext *ucontext, ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; } + if (attrs && uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_QP_BUF_UMEM)) { + struct irdma_mem_reg_req qp_req = {}; + struct ib_umem *umem; + u32 total; + + /* Legacy context does not provide depth info */ + if (!ucontext->use_raw_attrs) + return -EOPNOTSUPP; + + qp_req.sq_pages = DIV_ROUND_UP(ukinfo->sq_depth * IRDMA_QP_WQE_MIN_SIZE, + IRDMA_HW_PAGE_SIZE); + /* In case srq is used, this will be a single page */ + qp_req.rq_pages = DIV_ROUND_UP(ukinfo->rq_depth * IRDMA_QP_WQE_MIN_SIZE, + IRDMA_HW_PAGE_SIZE); + total = qp_req.sq_pages + qp_req.rq_pages + IRDMA_SHADOW_PGCNT; + + umem = ib_umem_get_attr(&iwdev->ibdev, attrs, + UVERBS_ATTR_CREATE_QP_BUF_UMEM, + total << IRDMA_HW_PAGE_SHIFT, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) + return PTR_ERR(umem); + + iwqp->iwpbl = irdma_setup_q_pbl_from_umem(iwdev, umem, + iwqp->ibqp.pd, + IRDMA_MEMREG_TYPE_QP, + &qp_req); + if (IS_ERR(iwqp->iwpbl)) { + ret = PTR_ERR(iwqp->iwpbl); + iwqp->iwpbl = NULL; + ib_umem_release(umem); + return ret; + } + } else { + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); + iwqp->iwpbl = irdma_get_pbl((unsigned long)req->user_wqe_bufs, + &ucontext->qp_reg_mem_list); + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + + if (!iwqp->iwpbl) { + ret = -ENODATA; + ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); + return ret; + } + } + irdma_setup_virt_qp(iwdev, iwqp, info); return 0; @@ -3460,6 +3501,7 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, } static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, + struct ib_device *ibdev, struct ib_pd *pd, u64 virt, enum irdma_memreg_type reg_type) { @@ -3475,7 +3517,7 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, iwpbl->iwmr = iwmr; iwmr->region = region; iwmr->ibmr.pd = pd; - iwmr->ibmr.device = pd->device; + iwmr->ibmr.device = ibdev; iwmr->ibmr.iova = virt; iwmr->type = reg_type; iwmr->len = region->length; @@ -3503,6 +3545,68 @@ static void irdma_free_iwmr(struct irdma_mr *iwmr) kfree(iwmr); } +static struct irdma_pbl *irdma_setup_q_pbl_from_umem(struct irdma_device *iwdev, + struct ib_umem *region, + struct ib_pd *pd, + enum irdma_memreg_type reg_type, + struct irdma_mem_reg_req *req) +{ + struct irdma_pbl *iwpbl; + struct irdma_mr *iwmr; + u32 total; + u8 lvl; + int err; + + /* Catch page not starting on OS page boundary */ + if (ib_umem_offset(region)) + return ERR_PTR(-EINVAL); + + /* iWarp: Reject queue dma-buf as it is not supported */ + if (!rdma_protocol_roce(&iwdev->ibdev, 1) && region->is_dmabuf) + return ERR_PTR(-EINVAL); + + iwmr = irdma_alloc_iwmr(region, &iwdev->ibdev, pd, region->address, reg_type); + if (IS_ERR(iwmr)) + return ERR_CAST(iwmr); + + iwpbl = &iwmr->iwpbl; + + switch (reg_type) { + case IRDMA_MEMREG_TYPE_QP: + total = req->sq_pages + req->rq_pages + IRDMA_SHADOW_PGCNT; + lvl = (req->sq_pages + req->rq_pages) > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; + break; + case IRDMA_MEMREG_TYPE_CQ: + total = req->cq_pages + + ((iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & + IRDMA_FEATURE_CQ_RESIZE) ? 0 : IRDMA_SHADOW_PGCNT); + lvl = req->cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; + break; + case IRDMA_MEMREG_TYPE_SRQ: + total = req->rq_pages + IRDMA_SHADOW_PGCNT; + lvl = req->rq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; + break; + default: + err = -EINVAL; + goto err_free_mr; + } + + if (total > iwmr->page_cnt) { + err = -EINVAL; + goto err_free_mr; + } + + err = irdma_handle_q_mem(iwdev, req, iwpbl, lvl); + if (err) + goto err_free_mr; + + return iwpbl; + +err_free_mr: + irdma_free_iwmr(iwmr); + return ERR_PTR(err); +} + static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req, struct ib_udata *udata, struct irdma_mr *iwmr) @@ -3648,7 +3752,7 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, return (struct ib_mr *)region; } - iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); + iwmr = irdma_alloc_iwmr(region, pd->device, pd, virt, req.reg_type); if (IS_ERR(iwmr)) { ib_umem_release(region); return (struct ib_mr *)iwmr; @@ -3747,7 +3851,7 @@ static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start, return ERR_CAST(umem_dmabuf); } - iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd, virt, IRDMA_MEMREG_TYPE_MEM); + iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd->device, pd, virt, IRDMA_MEMREG_TYPE_MEM); if (IS_ERR(iwmr)) { err = PTR_ERR(iwmr); goto err_release; -- 2.56.0.317.g454f8302e1-goog
