The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=259a763e343d36a6030e0bba5ae7ed05c123c2fe
commit 259a763e343d36a6030e0bba5ae7ed05c123c2fe Author: John Baldwin <[email protected]> AuthorDate: 2025-12-05 20:56:35 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2026-06-22 18:36:52 +0000 OFED: Implement ib_process_cq_direct This is largely pulled from the original Linux commit to add cq.c. Note that irq_poll is still not supported, but polling should now be possible whereas it wasn't really before. Tested by: Wafa Hamzah <[email protected]> (mlx5_ib) Tested by: John Baldwin <[email protected]> (iw_cxgbe) Obtained from: Linux commit 14d3a3b2498edadec344cb11e60e66091f5daf63 Sponsored by: Chelsio Communications --- sys/ofed/drivers/infiniband/core/ib_cq.c | 96 ++++++++++++++++++++++---------- sys/ofed/include/rdma/ib_verbs.h | 3 + 2 files changed, 70 insertions(+), 29 deletions(-) diff --git a/sys/ofed/drivers/infiniband/core/ib_cq.c b/sys/ofed/drivers/infiniband/core/ib_cq.c index d455f2c7c9ed..d6c897392022 100644 --- a/sys/ofed/drivers/infiniband/core/ib_cq.c +++ b/sys/ofed/drivers/infiniband/core/ib_cq.c @@ -39,42 +39,73 @@ #include <rdma/ib_verbs.h> -#define IB_CQ_POLL_MAX 16 -/* maximum number of completions per poll loop */ -#define IB_CQ_POLL_BUDGET 65536 -#define IB_CQ_POLL_FLAGS (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) +/* # of WCs to poll for with a single call to ib_poll_cq */ +#define IB_POLL_BATCH 16 -static void -ib_cq_poll_work(struct work_struct *work) +/* # of WCs to iterate over before yielding */ +#define IB_POLL_BUDGET_WORKQUEUE 65536 + +#define IB_POLL_FLAGS \ + (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) + +static int __ib_process_cq(struct ib_cq *cq, int budget) { - struct ib_wc ib_wc[IB_CQ_POLL_MAX]; - struct ib_cq *cq = container_of(work, struct ib_cq, work); - int total = 0; - int i; - int n; + int i, n, completed = 0; - while (1) { - n = ib_poll_cq(cq, IB_CQ_POLL_MAX, ib_wc); + while ((n = ib_poll_cq(cq, IB_POLL_BATCH, cq->wc)) > 0) { for (i = 0; i < n; i++) { - struct ib_wc *wc = ib_wc + i; + struct ib_wc *wc = &cq->wc[i]; - if (wc->wr_cqe != NULL) + if (wc->wr_cqe) wc->wr_cqe->done(cq, wc); - } - - if (n != IB_CQ_POLL_MAX) { - if (ib_req_notify_cq(cq, IB_CQ_POLL_FLAGS) > 0) - break; else - return; + WARN_ON_ONCE(wc->status == IB_WC_SUCCESS); } - total += n; - if (total >= IB_CQ_POLL_BUDGET) + + completed += n; + + if (n != IB_POLL_BATCH || + (budget != -1 && completed >= budget)) break; } - /* give other work structs a chance */ - queue_work(ib_comp_wq, &cq->work); + return completed; +} + +/** + * ib_process_direct_cq - process a CQ in caller context + * @cq: CQ to process + * @budget: number of CQEs to poll for + * + * This function is used to process all outstanding CQ entries on a + * %IB_POLL_DIRECT CQ. It does not offload CQ processing to a different + * context and does not ask for completion interrupts from the HCA. + * + * Note: for compatibility reasons -1 can be passed in %budget for unlimited + * polling. Do not use this feature in new code, it will be removed soon. + */ +int ib_process_cq_direct(struct ib_cq *cq, int budget) +{ + WARN_ON_ONCE(cq->poll_ctx != IB_POLL_DIRECT); + + return __ib_process_cq(cq, budget); +} +EXPORT_SYMBOL(ib_process_cq_direct); + +static void ib_cq_completion_direct(struct ib_cq *cq, void *private) +{ + WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq); +} + +static void ib_cq_poll_work(struct work_struct *work) +{ + struct ib_cq *cq = container_of(work, struct ib_cq, work); + int completed; + + completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE); + if (completed >= IB_POLL_BUDGET_WORKQUEUE || + ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) + queue_work(ib_comp_wq, &cq->work); } static void @@ -94,7 +125,7 @@ __ib_alloc_cq_user(struct ib_device *dev, void *private, .comp_vector = comp_vector, }; struct ib_cq *cq; - int ret; + int ret = -ENOMEM; /* * Check for invalid parameters early on to avoid @@ -118,13 +149,17 @@ __ib_alloc_cq_user(struct ib_device *dev, void *private, cq->poll_ctx = poll_ctx; atomic_set(&cq->usecnt, 0); + cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL); + if (!cq->wc) + goto out_free_cq; + ret = dev->create_cq(cq, &cq_attr, NULL); if (ret) - goto out_free_cq; + goto out_free_wc; - switch (poll_ctx) { + switch (cq->poll_ctx) { case IB_POLL_DIRECT: - cq->comp_handler = NULL; /* no hardware completions */ + cq->comp_handler = ib_cq_completion_direct; break; case IB_POLL_SOFTIRQ: case IB_POLL_WORKQUEUE: @@ -137,6 +172,8 @@ __ib_alloc_cq_user(struct ib_device *dev, void *private, } return (cq); +out_free_wc: + kfree(cq->wc); out_free_cq: kfree(cq); return (ERR_PTR(ret)); @@ -161,6 +198,7 @@ ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) break; } + kfree(cq->wc); cq->device->destroy_cq(cq, udata); kfree(cq); } diff --git a/sys/ofed/include/rdma/ib_verbs.h b/sys/ofed/include/rdma/ib_verbs.h index 086251183a55..8929cf2fe0b1 100644 --- a/sys/ofed/include/rdma/ib_verbs.h +++ b/sys/ofed/include/rdma/ib_verbs.h @@ -1472,6 +1472,7 @@ struct ib_cq { void *cq_context; int cqe; atomic_t usecnt; /* count number of work queues */ + struct ib_wc *wc; enum ib_poll_context poll_ctx; struct work_struct work; }; @@ -3354,6 +3355,8 @@ static inline void ib_free_cq(struct ib_cq *cq) ib_free_cq_user(cq, NULL); } +int ib_process_cq_direct(struct ib_cq *cq, int budget); + /** * ib_create_cq - Creates a CQ on the specified device. * @device: The device on which to create the CQ.
