From: Leon Romanovsky <[email protected]> Allow users to supply their own umem.
Signed-off-by: Leon Romanovsky <[email protected]> --- drivers/infiniband/hw/qedr/main.c | 1 + drivers/infiniband/hw/qedr/verbs.c | 323 +++++++++++++++++++++---------------- drivers/infiniband/hw/qedr/verbs.h | 2 + 3 files changed, 188 insertions(+), 138 deletions(-) diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index ecdfeff3d44f..c6ca95983492 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -199,6 +199,7 @@ static const struct ib_device_ops qedr_dev_ops = { .alloc_ucontext = qedr_alloc_ucontext, .create_ah = qedr_create_ah, .create_cq = qedr_create_cq, + .create_user_cq = qedr_create_user_cq, .create_qp = qedr_create_qp, .create_srq = qedr_create_srq, .dealloc_pd = qedr_dealloc_pd, diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index cb06c5d894b8..10010ccf63b3 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -789,52 +789,33 @@ static int qedr_init_user_db_rec(struct ib_udata *udata, static inline int qedr_init_user_queue(struct ib_udata *udata, struct qedr_dev *dev, - struct qedr_userq *q, u64 buf_addr, - size_t buf_len, bool requires_db_rec, - int access, + struct qedr_userq *q, + bool requires_db_rec, int alloc_and_init) { u32 fw_pages; int rc; - q->buf_addr = buf_addr; - q->buf_len = buf_len; - q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); - if (IS_ERR(q->umem)) { - DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", - PTR_ERR(q->umem)); - return PTR_ERR(q->umem); - } - fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT); rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); if (rc) - goto err0; + return rc; if (alloc_and_init) { q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); - if (IS_ERR(q->pbl_tbl)) { - rc = PTR_ERR(q->pbl_tbl); - goto err0; - } + if (IS_ERR(q->pbl_tbl)) + return PTR_ERR(q->pbl_tbl); + qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, FW_PAGE_SHIFT); } else { q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); - if (!q->pbl_tbl) { - rc = -ENOMEM; - goto err0; - } + if (!q->pbl_tbl) + return -ENOMEM; } /* mmap the user address used to store doorbell data for recovery */ return qedr_init_user_db_rec(udata, dev, q, requires_db_rec); - -err0: - ib_umem_release(q->umem); - q->umem = NULL; - - return rc; } static inline void qedr_init_cq_params(struct qedr_cq *cq, @@ -899,8 +880,8 @@ int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) return 0; } -int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, - struct uverbs_attr_bundle *attrs) +int qedr_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct uverbs_attr_bundle *attrs) { struct ib_udata *udata = &attrs->driver_udata; struct ib_device *ibdev = ibcq->device; @@ -908,6 +889,104 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, udata, struct qedr_ucontext, ibucontext); struct qed_rdma_destroy_cq_out_params destroy_oparams; struct qed_rdma_destroy_cq_in_params destroy_iparams; + struct qedr_dev *dev = get_qedr_dev(ibdev); + struct qed_rdma_create_cq_in_params params; + struct qedr_create_cq_ureq ureq = {}; + int vector = attr->comp_vector; + int entries = attr->cqe; + struct qedr_cq *cq = get_qedr_cq(ibcq); + int chain_entries; + u32 db_offset; + int page_cnt; + u64 pbl_ptr; + u16 icid; + int rc; + + DP_DEBUG(dev, QEDR_MSG_INIT, + "create_cq: called from User Lib. entries=%d, vector=%d\n", + entries, vector); + + if (attr->flags) + return -EOPNOTSUPP; + + if (attr->cqe > QEDR_MAX_CQES) + return -EINVAL; + + chain_entries = qedr_align_cq_entries(entries); + chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); + + /* calc db offset. user will add DPI base, kernel will add db addr */ + db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); + + if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen))) + return -EINVAL; + + cq->cq_type = QEDR_CQ_TYPE_USER; + + cq->q.buf_addr = ureq.addr; + cq->q.buf_len = ureq.len; + if (!ibcq->umem) + ibcq->umem = ib_umem_get(&dev->ibdev, ureq.addr, ureq.len, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(ibcq->umem)) + return PTR_ERR(ibcq->umem); + cq->q.umem = ibcq->umem; + + rc = qedr_init_user_queue(udata, dev, &cq->q, true, 1); + if (rc) + return rc; + + pbl_ptr = cq->q.pbl_tbl->pa; + page_cnt = cq->q.pbl_info.num_pbes; + + cq->ibcq.cqe = chain_entries; + cq->q.db_addr = ctx->dpi_addr + db_offset; + + qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, + pbl_ptr, ¶ms); + + rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); + if (rc) + goto err1; + + cq->icid = icid; + cq->sig = QEDR_CQ_MAGIC_NUMBER; + spin_lock_init(&cq->cq_lock); + + rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); + if (rc) + goto err2; + + rc = qedr_db_recovery_add(dev, cq->q.db_addr, + &cq->q.db_rec_data->db_data, + DB_REC_WIDTH_64B, + DB_REC_USER); + if (rc) + goto err2; + + DP_DEBUG(dev, QEDR_MSG_CQ, + "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", + cq->icid, cq, params.cq_size); + + return 0; + +err2: + destroy_iparams.icid = cq->icid; + dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, + &destroy_oparams); +err1: + qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); + if (cq->q.db_mmap_entry) + rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); + return rc; +} + +int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct uverbs_attr_bundle *attrs) +{ + struct ib_device *ibdev = ibcq->device; + struct qed_rdma_destroy_cq_out_params destroy_oparams; + struct qed_rdma_destroy_cq_in_params destroy_iparams; struct qed_chain_init_params chain_params = { .mode = QED_CHAIN_MODE_PBL, .intended_use = QED_CHAIN_USE_TO_CONSUME, @@ -916,7 +995,6 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, }; struct qedr_dev *dev = get_qedr_dev(ibdev); struct qed_rdma_create_cq_in_params params; - struct qedr_create_cq_ureq ureq = {}; int vector = attr->comp_vector; int entries = attr->cqe; struct qedr_cq *cq = get_qedr_cq(ibcq); @@ -928,18 +1006,14 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, int rc; DP_DEBUG(dev, QEDR_MSG_INIT, - "create_cq: called from %s. entries=%d, vector=%d\n", - udata ? "User Lib" : "Kernel", entries, vector); + "create_cq: called from Kernel. entries=%d, vector=%d\n", + entries, vector); if (attr->flags) return -EOPNOTSUPP; - if (entries > QEDR_MAX_CQES) { - DP_ERR(dev, - "create cq: the number of entries %d is too high. Must be equal or below %d.\n", - entries, QEDR_MAX_CQES); + if (attr->cqe > QEDR_MAX_CQES) return -EINVAL; - } chain_entries = qedr_align_cq_entries(entries); chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); @@ -948,47 +1022,18 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, /* calc db offset. user will add DPI base, kernel will add db addr */ db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); - if (udata) { - if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), - udata->inlen))) { - DP_ERR(dev, - "create cq: problem copying data from user space\n"); - goto err0; - } + cq->cq_type = QEDR_CQ_TYPE_KERNEL; - if (!ureq.len) { - DP_ERR(dev, - "create cq: cannot create a cq with 0 entries\n"); - goto err0; - } - - cq->cq_type = QEDR_CQ_TYPE_USER; - - rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, - ureq.len, true, IB_ACCESS_LOCAL_WRITE, - 1); - if (rc) - goto err0; - - pbl_ptr = cq->q.pbl_tbl->pa; - page_cnt = cq->q.pbl_info.num_pbes; - - cq->ibcq.cqe = chain_entries; - cq->q.db_addr = ctx->dpi_addr + db_offset; - } else { - cq->cq_type = QEDR_CQ_TYPE_KERNEL; + rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl, + &chain_params); + if (rc) + return rc; - rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl, - &chain_params); - if (rc) - goto err0; + page_cnt = qed_chain_get_page_cnt(&cq->pbl); + pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); + cq->ibcq.cqe = cq->pbl.capacity; - page_cnt = qed_chain_get_page_cnt(&cq->pbl); - pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); - cq->ibcq.cqe = cq->pbl.capacity; - } - - qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, + qedr_init_cq_params(cq, NULL, dev, vector, chain_entries, page_cnt, pbl_ptr, ¶ms); rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); @@ -999,37 +1044,23 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->sig = QEDR_CQ_MAGIC_NUMBER; spin_lock_init(&cq->cq_lock); - if (udata) { - rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); - if (rc) - goto err2; - - rc = qedr_db_recovery_add(dev, cq->q.db_addr, - &cq->q.db_rec_data->db_data, - DB_REC_WIDTH_64B, - DB_REC_USER); - if (rc) - goto err2; + /* Generate doorbell address. */ + cq->db.data.icid = cq->icid; + cq->db_addr = dev->db_addr + db_offset; + cq->db.data.params = DB_AGG_CMD_MAX << + RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; - } else { - /* Generate doorbell address. */ - cq->db.data.icid = cq->icid; - cq->db_addr = dev->db_addr + db_offset; - cq->db.data.params = DB_AGG_CMD_MAX << - RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; - - /* point to the very last element, passing it we will toggle */ - cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); - cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; - cq->latest_cqe = NULL; - consume_cqe(cq); - cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); + /* point to the very last element, passing it we will toggle */ + cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); + cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; + cq->latest_cqe = NULL; + consume_cqe(cq); + cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); - rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, - DB_REC_WIDTH_64B, DB_REC_KERNEL); - if (rc) - goto err2; - } + rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, + DB_REC_WIDTH_64B, DB_REC_KERNEL); + if (rc) + goto err2; DP_DEBUG(dev, QEDR_MSG_CQ, "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", @@ -1042,16 +1073,8 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); err1: - if (udata) { - qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); - ib_umem_release(cq->q.umem); - if (cq->q.db_mmap_entry) - rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); - } else { - dev->ops->common->chain_free(dev->cdev, &cq->pbl); - } -err0: - return -EINVAL; + dev->ops->common->chain_free(dev->cdev, &cq->pbl); + return rc; } #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) @@ -1081,7 +1104,6 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (udata) { qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); - ib_umem_release(cq->q.umem); if (cq->q.db_rec_data) { qedr_db_recovery_del(dev, cq->q.db_addr, @@ -1472,26 +1494,33 @@ static int qedr_init_srq_user_params(struct ib_udata *udata, struct scatterlist *sg; int rc; - rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, - ureq->srq_len, false, access, 1); + srq->usrq.buf_addr = ureq->srq_addr; + srq->usrq.buf_len = ureq->srq_len; + srq->usrq.umem = ib_umem_get(&srq->dev->ibdev, ureq->srq_addr, + ureq->srq_len, access); + if (IS_ERR(srq->usrq.umem)) + return PTR_ERR(srq->usrq.umem); + + rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, false, 1); if (rc) - return rc; + goto err_umem; srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, sizeof(struct rdma_srq_producers), access); if (IS_ERR(srq->prod_umem)) { + rc = PTR_ERR(srq->prod_umem); qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); - ib_umem_release(srq->usrq.umem); - DP_ERR(srq->dev, - "create srq: failed ib_umem_get for producer, got %ld\n", - PTR_ERR(srq->prod_umem)); - return PTR_ERR(srq->prod_umem); + goto err_umem; } sg = srq->prod_umem->sgt_append.sgt.sgl; srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); return 0; + +err_umem: + ib_umem_release(srq->usrq.umem); + return rc; } static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, @@ -1870,27 +1899,34 @@ static int qedr_create_user_qp(struct qedr_dev *dev, if (qedr_qp_has_sq(qp)) { /* SQ - read access only (0) */ - rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, - ureq.sq_len, true, 0, alloc_and_init); + qp->usq.buf_addr = ureq.sq_addr; + qp->usq.buf_len = ureq.sq_len; + qp->usq.umem = ib_umem_get(&dev->ibdev, ureq.sq_addr, + ureq.sq_len, 0); + if (IS_ERR(qp->usq.umem)) + return PTR_ERR(qp->usq.umem); + + rc = qedr_init_user_queue(udata, dev, &qp->usq, true, + alloc_and_init); if (rc) - return rc; + goto err_sq_umem; } if (qedr_qp_has_rq(qp)) { /* RQ - read access only (0) */ - rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, - ureq.rq_len, true, 0, alloc_and_init); - if (rc) { - ib_umem_release(qp->usq.umem); - qp->usq.umem = NULL; - if (rdma_protocol_roce(&dev->ibdev, 1)) { - qedr_free_pbl(dev, &qp->usq.pbl_info, - qp->usq.pbl_tbl); - } else { - kfree(qp->usq.pbl_tbl); - } - return rc; + qp->urq.buf_addr = ureq.rq_addr; + qp->urq.buf_len = ureq.rq_len; + qp->urq.umem = ib_umem_get(&dev->ibdev, ureq.rq_addr, + ureq.rq_len, 0); + if (IS_ERR(qp->urq.umem)) { + rc = PTR_ERR(qp->urq.umem); + goto err_rq_umem; } + + rc = qedr_init_user_queue(udata, dev, &qp->urq, true, + alloc_and_init); + if (rc) + goto err_rq_umem2; } memset(&in_params, 0, sizeof(in_params)); @@ -1989,6 +2025,17 @@ static int qedr_create_user_qp(struct qedr_dev *dev, err1: qedr_cleanup_user(dev, ctx, qp); return rc; + +err_rq_umem2: + ib_umem_release(qp->urq.umem); +err_rq_umem: + if (rdma_protocol_roce(&dev->ibdev, 1)) + qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); + else + kfree(qp->usq.pbl_tbl); +err_sq_umem: + ib_umem_release(qp->usq.umem); + return rc; } static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h index 62420a15101b..292d77df562d 100644 --- a/drivers/infiniband/hw/qedr/verbs.h +++ b/drivers/infiniband/hw/qedr/verbs.h @@ -53,6 +53,8 @@ int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata); int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs); +int qedr_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct uverbs_attr_bundle *attrs); int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); int qedr_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs, -- 2.52.0
