> Subject: [PATCH rdma-next 12/13] RDMA/mana_ib: polling of CQs for GSI/UD > > From: Konstantin Taranov <[email protected]> > > Add polling for the kernel CQs. > Process completion events for UD/GSI QPs. > > Signed-off-by: Konstantin Taranov <[email protected]> > Reviewed-by: Shiraz Saleem <[email protected]>
Reviewed-by: Long Li <[email protected]> > --- > drivers/infiniband/hw/mana/cq.c | 135 ++++++++++++++++++ > drivers/infiniband/hw/mana/device.c | 1 + > drivers/infiniband/hw/mana/mana_ib.h | 32 +++++ > drivers/infiniband/hw/mana/qp.c | 33 +++++ > .../net/ethernet/microsoft/mana/gdma_main.c | 1 + > 5 files changed, 202 insertions(+) > > diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c > index 82f1462..5c325ef 100644 > --- a/drivers/infiniband/hw/mana/cq.c > +++ b/drivers/infiniband/hw/mana/cq.c > @@ -90,6 +90,10 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct > ib_cq_init_attr *attr, > } > } > > + spin_lock_init(&cq->cq_lock); > + INIT_LIST_HEAD(&cq->list_send_qp); > + INIT_LIST_HEAD(&cq->list_recv_qp); > + > return 0; > > err_remove_cq_cb: > @@ -180,3 +184,134 @@ int mana_ib_arm_cq(struct ib_cq *ibcq, enum > ib_cq_notify_flags flags) > mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); > return 0; > } > + > +static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct > +gdma_comp *cqe) { > + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe- > >cqe_data; > + struct gdma_queue *wq = qp- > >ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; > + struct ud_sq_shadow_wqe *shadow_wqe; > + > + shadow_wqe = shadow_queue_get_next_to_complete(&qp- > >shadow_sq); > + if (!shadow_wqe) > + return; > + > + shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; > + > + wq->tail += shadow_wqe->header.posted_wqe_size; > + shadow_queue_advance_next_to_complete(&qp->shadow_sq); > +} > + > +static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct > +gdma_comp *cqe) { > + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe- > >cqe_data; > + struct gdma_queue *wq = qp- > >ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; > + struct ud_rq_shadow_wqe *shadow_wqe; > + > + shadow_wqe = shadow_queue_get_next_to_complete(&qp- > >shadow_rq); > + if (!shadow_wqe) > + return; > + > + shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; > + shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; > + shadow_wqe->header.error_code = IB_WC_SUCCESS; > + > + wq->tail += shadow_wqe->header.posted_wqe_size; > + shadow_queue_advance_next_to_complete(&qp->shadow_rq); > +} > + > +static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp > +*cqe) { > + struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, > +cqe->is_sq); > + > + if (!qp) > + return; > + > + if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) > { > + if (cqe->is_sq) > + handle_ud_sq_cqe(qp, cqe); > + else > + handle_ud_rq_cqe(qp, cqe); > + } > + > + mana_put_qp_ref(qp); > +} > + > +static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc > *wc, > + const struct shadow_wqe_header > *shadow_wqe) { > + const struct ud_rq_shadow_wqe *ud_wqe = (const struct > ud_rq_shadow_wqe > +*)shadow_wqe; > + > + wc->wr_id = shadow_wqe->wr_id; > + wc->status = shadow_wqe->error_code; > + wc->opcode = shadow_wqe->opcode; > + wc->vendor_err = shadow_wqe->error_code; > + wc->wc_flags = 0; > + wc->qp = &qp->ibqp; > + wc->pkey_index = 0; > + > + if (shadow_wqe->opcode == IB_WC_RECV) { > + wc->byte_len = ud_wqe->byte_len; > + wc->src_qp = ud_wqe->src_qpn; > + wc->wc_flags |= IB_WC_GRH; > + } > +} > + > +static int mana_process_completions(struct mana_ib_cq *cq, int nwc, > +struct ib_wc *wc) { > + struct shadow_wqe_header *shadow_wqe; > + struct mana_ib_qp *qp; > + int wc_index = 0; > + > + /* process send shadow queue completions */ > + list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { > + while ((shadow_wqe = > shadow_queue_get_next_to_consume(&qp->shadow_sq)) > + != NULL) { > + if (wc_index >= nwc) > + goto out; > + > + fill_verbs_from_shadow_wqe(qp, &wc[wc_index], > shadow_wqe); > + shadow_queue_advance_consumer(&qp->shadow_sq); > + wc_index++; > + } > + } > + > + /* process recv shadow queue completions */ > + list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { > + while ((shadow_wqe = > shadow_queue_get_next_to_consume(&qp->shadow_rq)) > + != NULL) { > + if (wc_index >= nwc) > + goto out; > + > + fill_verbs_from_shadow_wqe(qp, &wc[wc_index], > shadow_wqe); > + shadow_queue_advance_consumer(&qp->shadow_rq); > + wc_index++; > + } > + } > + > +out: > + return wc_index; > +} > + > +int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc > +*wc) { > + struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); > + struct mana_ib_dev *mdev = container_of(ibcq->device, struct > mana_ib_dev, ib_dev); > + struct gdma_queue *queue = cq->queue.kmem; > + struct gdma_comp gdma_cqe; > + unsigned long flags; > + int num_polled = 0; > + int comp_read, i; > + > + spin_lock_irqsave(&cq->cq_lock, flags); > + for (i = 0; i < num_entries; i++) { > + comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); > + if (comp_read < 1) > + break; > + mana_handle_cqe(mdev, &gdma_cqe); > + } > + > + num_polled = mana_process_completions(cq, num_entries, wc); > + spin_unlock_irqrestore(&cq->cq_lock, flags); > + > + return num_polled; > +} > diff --git a/drivers/infiniband/hw/mana/device.c > b/drivers/infiniband/hw/mana/device.c > index 63e12c3..97502bc 100644 > --- a/drivers/infiniband/hw/mana/device.c > +++ b/drivers/infiniband/hw/mana/device.c > @@ -40,6 +40,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { > .mmap = mana_ib_mmap, > .modify_qp = mana_ib_modify_qp, > .modify_wq = mana_ib_modify_wq, > + .poll_cq = mana_ib_poll_cq, > .post_recv = mana_ib_post_recv, > .post_send = mana_ib_post_send, > .query_device = mana_ib_query_device, > diff --git a/drivers/infiniband/hw/mana/mana_ib.h > b/drivers/infiniband/hw/mana/mana_ib.h > index 5e4ca55..cd771af 100644 > --- a/drivers/infiniband/hw/mana/mana_ib.h > +++ b/drivers/infiniband/hw/mana/mana_ib.h > @@ -127,6 +127,10 @@ struct mana_ib_mr { struct mana_ib_cq { > struct ib_cq ibcq; > struct mana_ib_queue queue; > + /* protects CQ polling */ > + spinlock_t cq_lock; > + struct list_head list_send_qp; > + struct list_head list_recv_qp; > int cqe; > u32 comp_vector; > mana_handle_t cq_handle; > @@ -169,6 +173,8 @@ struct mana_ib_qp { > /* The port on the IB device, starting with 1 */ > u32 port; > > + struct list_head cq_send_list; > + struct list_head cq_recv_list; > struct shadow_queue shadow_rq; > struct shadow_queue shadow_sq; > > @@ -435,6 +441,31 @@ struct rdma_send_oob { > }; > }; /* HW DATA */ > > +struct mana_rdma_cqe { > + union { > + struct { > + u8 cqe_type; > + u8 data[GDMA_COMP_DATA_SIZE - 1]; > + }; > + struct { > + u32 cqe_type : 8; > + u32 vendor_error : 9; > + u32 reserved1 : 15; > + u32 sge_offset : 5; > + u32 tx_wqe_offset : 27; > + } ud_send; > + struct { > + u32 cqe_type : 8; > + u32 reserved1 : 24; > + u32 msg_len; > + u32 src_qpn : 24; > + u32 reserved2 : 8; > + u32 imm_data; > + u32 rx_wqe_offset; > + } ud_recv; > + }; > +}; /* HW DATA */ > + > static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) { > return mdev->gdma_dev->gdma_context; > @@ -602,5 +633,6 @@ int mana_ib_post_recv(struct ib_qp *ibqp, const struct > ib_recv_wr *wr, int mana_ib_post_send(struct ib_qp *ibqp, const struct > ib_send_wr *wr, > const struct ib_send_wr **bad_wr); > > +int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc > +*wc); > int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); > #endif > diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c > index 2528046..b05e64b 100644 > --- a/drivers/infiniband/hw/mana/qp.c > +++ b/drivers/infiniband/hw/mana/qp.c > @@ -600,6 +600,36 @@ destroy_queues: > return err; > } > > +static void mana_add_qp_to_cqs(struct mana_ib_qp *qp) { > + struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct > mana_ib_cq, ibcq); > + struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct > mana_ib_cq, ibcq); > + unsigned long flags; > + > + spin_lock_irqsave(&send_cq->cq_lock, flags); > + list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); > + spin_unlock_irqrestore(&send_cq->cq_lock, flags); > + > + spin_lock_irqsave(&recv_cq->cq_lock, flags); > + list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); > + spin_unlock_irqrestore(&recv_cq->cq_lock, flags); } > + > +static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp) { > + struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct > mana_ib_cq, ibcq); > + struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct > mana_ib_cq, ibcq); > + unsigned long flags; > + > + spin_lock_irqsave(&send_cq->cq_lock, flags); > + list_del(&qp->cq_send_list); > + spin_unlock_irqrestore(&send_cq->cq_lock, flags); > + > + spin_lock_irqsave(&recv_cq->cq_lock, flags); > + list_del(&qp->cq_recv_list); > + spin_unlock_irqrestore(&recv_cq->cq_lock, flags); } > + > static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, > struct ib_qp_init_attr *attr, struct ib_udata > *udata) { @@ -654,6 +684,8 @@ static int mana_ib_create_ud_qp(struct ib_qp > *ibqp, struct ib_pd *ibpd, > if (err) > goto destroy_qp; > > + mana_add_qp_to_cqs(qp); > + > return 0; > > destroy_qp: > @@ -840,6 +872,7 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp > *qp, struct ib_udata *udata) > container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); > int i; > > + mana_remove_qp_from_cqs(qp); > mana_table_remove_qp(mdev, qp); > > destroy_shadow_queue(&qp->shadow_rq); > diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c > b/drivers/net/ethernet/microsoft/mana/gdma_main.c > index 823f7e7..2da15d9 100644 > --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c > +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c > @@ -1222,6 +1222,7 @@ int mana_gd_poll_cq(struct gdma_queue *cq, struct > gdma_comp *comp, int num_cqe) > > return cqe_idx; > } > +EXPORT_SYMBOL_NS(mana_gd_poll_cq, NET_MANA); > > static irqreturn_t mana_gd_intr(int irq, void *arg) { > -- > 2.43.0
