The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=6c858dab48d67c1de0a3827d7f13600de49f9edb
commit 6c858dab48d67c1de0a3827d7f13600de49f9edb Author: Steve Wise <[email protected]> AuthorDate: 2017-06-23 20:41:08 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2026-09-15 14:22:38 +0000 iw_cxgbe/libcxgb4: Support for RDMA WRITE_WITH_IMMEDIATE requests Sponsored by: Chelsio Communications --- contrib/ofed/libcxgb4/cq.c | 9 ++++++++- contrib/ofed/libcxgb4/cxgb4-abi.h | 3 ++- contrib/ofed/libcxgb4/qp.c | 12 +++++++++++- contrib/ofed/libcxgb4/t4.h | 11 +++++++++-- contrib/ofed/libcxgb4/verbs.c | 2 ++ sys/dev/cxgbe/iw_cxgbe/cq.c | 25 ++++++++++++++++++++++--- sys/dev/cxgbe/iw_cxgbe/qp.c | 14 +++++++++++++- sys/dev/cxgbe/iw_cxgbe/t4.h | 8 +++++++- sys/dev/cxgbe/iw_cxgbe/user.h | 4 ++++ 9 files changed, 78 insertions(+), 10 deletions(-) diff --git a/contrib/ofed/libcxgb4/cq.c b/contrib/ofed/libcxgb4/cq.c index 1b79cb5d21a1..4841535f8878 100644 --- a/contrib/ofed/libcxgb4/cq.c +++ b/contrib/ofed/libcxgb4/cq.c @@ -616,10 +616,17 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ibv_wc *wc) wc->byte_len = CQE_LEN(&cqe); else wc->byte_len = 0; - wc->opcode = IBV_WC_RECV; + if (CQE_OPCODE(&cqe) == FW_RI_WRITE_IMMEDIATE) { + wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM; + wc->imm_data = CQE_IMM_DATA(&cqe); + wc->wc_flags |= IBV_WC_WITH_IMM; + } else { + wc->opcode = IBV_WC_RECV; + } } else { switch (CQE_OPCODE(&cqe)) { case FW_RI_RDMA_WRITE: + case FW_RI_WRITE_IMMEDIATE: wc->opcode = IBV_WC_RDMA_WRITE; break; case FW_RI_READ_REQ: diff --git a/contrib/ofed/libcxgb4/cxgb4-abi.h b/contrib/ofed/libcxgb4/cxgb4-abi.h index a42012c1ff0e..8c47b832d0f8 100644 --- a/contrib/ofed/libcxgb4/cxgb4-abi.h +++ b/contrib/ofed/libcxgb4/cxgb4-abi.h @@ -64,7 +64,8 @@ struct c4iw_create_cq_resp { }; enum { - C4IW_QPF_ONCHIP = (1<<0), + C4IW_QPF_ONCHIP = (1<<0), + C4IW_QPF_WRITE_W_IMM = (1<<1), }; struct c4iw_create_qp_resp_v0 { diff --git a/contrib/ofed/libcxgb4/qp.c b/contrib/ofed/libcxgb4/qp.c index 351cf621b82d..6d899c07ddae 100644 --- a/contrib/ofed/libcxgb4/qp.c +++ b/contrib/ofed/libcxgb4/qp.c @@ -213,7 +213,10 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, if (wr->num_sge > T4_MAX_SEND_SGE) return -EINVAL; - wqe->write.immd_data = 0; + if (wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM) + wqe->write.immd_data = wr->imm_data; + else + wqe->write.immd_data = 0; wqe->write.stag_sink = htobe32(wr->wr.rdma.rkey); wqe->write.to_sink = htobe64(wr->wr.rdma.remote_addr); if (wr->num_sge) { @@ -355,6 +358,13 @@ int c4iw_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, swsqe->opcode = FW_RI_SEND; err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); break; + case IBV_WR_RDMA_WRITE_WITH_IMM: + if (unlikely(!(qhp->wq.sq.flags & T4_SQ_WRITE_W_IMM))) { + err = -ENOSYS; + break; + } + fw_flags |= FW_RI_RDMA_WRITE_WITH_IMMEDIATE; + /*FALLTHROUGH*/ case IBV_WR_RDMA_WRITE: INC_STAT(write); fw_opcode = FW_RI_RDMA_WRITE_WR; diff --git a/contrib/ofed/libcxgb4/t4.h b/contrib/ofed/libcxgb4/t4.h index 87c2d4ee82df..921af87424fe 100644 --- a/contrib/ofed/libcxgb4/t4.h +++ b/contrib/ofed/libcxgb4/t4.h @@ -213,8 +213,13 @@ struct t4_cqe { __be32 wrid_hi; __be32 wrid_low; } gen; + struct { + __be32 mo; + __be32 msn; + __u64 imm_data; + } imm_data_rcqe; } u; - __be64 reserved[4]; + __be64 reserved[3]; __be64 bits_type_ts; }; @@ -264,6 +269,7 @@ struct t4_cqe { /* used for RQ completion processing */ #define CQE_WRID_STAG(x) (be32toh((x)->u.rcqe.stag)) #define CQE_WRID_MSN(x) (be32toh((x)->u.rcqe.msn)) +#define CQE_IMM_DATA(x) ((x)->u.imm_data_rcqe.imm_data) /* used for SQ completion processing */ #define CQE_WRID_SQ_IDX(x) (x)->u.scqe.cidx @@ -305,7 +311,8 @@ struct t4_swsqe { }; enum { - T4_SQ_ONCHIP = (1<<0), + T4_SQ_ONCHIP = (1<<0), + T4_SQ_WRITE_W_IMM = (1<<2) }; struct t4_sq { diff --git a/contrib/ofed/libcxgb4/verbs.c b/contrib/ofed/libcxgb4/verbs.c index 6188134146a9..64301a68a6b5 100644 --- a/contrib/ofed/libcxgb4/verbs.c +++ b/contrib/ofed/libcxgb4/verbs.c @@ -455,6 +455,8 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd, qhp->wq.sq.size = resp.sq_size; qhp->wq.sq.memsize = resp.sq_memsize; qhp->wq.sq.flags = resp.flags & C4IW_QPF_ONCHIP ? T4_SQ_ONCHIP : 0; + if (resp.flags & C4IW_QPF_WRITE_W_IMM) + qhp->wq.sq.flags |= T4_SQ_WRITE_W_IMM; qhp->wq.sq.flush_cidx = -1; qhp->wq.rq.msn = 1; qhp->wq.rq.qid = resp.rqid; diff --git a/sys/dev/cxgbe/iw_cxgbe/cq.c b/sys/dev/cxgbe/iw_cxgbe/cq.c index 02ee18ebd60c..65351dbfaafb 100644 --- a/sys/dev/cxgbe/iw_cxgbe/cq.c +++ b/sys/dev/cxgbe/iw_cxgbe/cq.c @@ -754,15 +754,34 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) wc->byte_len = CQE_LEN(&cqe); else wc->byte_len = 0; - wc->opcode = IB_WC_RECV; - if (CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_INV || - CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) { + + switch (CQE_OPCODE(&cqe)) { + case FW_RI_SEND: + case C4IW_DRAIN_OPCODE: + wc->opcode = IB_WC_RECV; + break; + case FW_RI_SEND_WITH_INV: + case FW_RI_SEND_WITH_SE_INV: + wc->opcode = IB_WC_RECV; wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe); wc->wc_flags |= IB_WC_WITH_INVALIDATE; c4iw_invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey); + break; + case FW_RI_WRITE_IMMEDIATE: + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + wc->ex.imm_data = CQE_IMM_DATA(&cqe); + wc->wc_flags |= IB_WC_WITH_IMM; + break; + default: + printf("Unexpected opcode %d " + "in the CQE received for QPID=0x%0x\n", + CQE_OPCODE(&cqe), CQE_QPID(&cqe)); + ret = -EINVAL; + goto out; } } else { switch (CQE_OPCODE(&cqe)) { + case FW_RI_WRITE_IMMEDIATE: case FW_RI_RDMA_WRITE: wc->opcode = IB_WC_RDMA_WRITE; break; diff --git a/sys/dev/cxgbe/iw_cxgbe/qp.c b/sys/dev/cxgbe/iw_cxgbe/qp.c index bfe2f9278abd..16aed5e423f0 100644 --- a/sys/dev/cxgbe/iw_cxgbe/qp.c +++ b/sys/dev/cxgbe/iw_cxgbe/qp.c @@ -481,7 +481,10 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, if (wr->num_sge > T4_MAX_SEND_SGE) return -EINVAL; - wqe->write.immd_data = 0; + if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) + wqe->write.immd_data = wr->ex.imm_data; + else + wqe->write.immd_data = 0; wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey); wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr); if (wr->num_sge) { @@ -829,6 +832,13 @@ int c4iw_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, swsqe->opcode = FW_RI_SEND_WITH_INV; err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); break; + case IB_WR_RDMA_WRITE_WITH_IMM: + if (unlikely(!qhp->rhp->rdev.adap->params.write_w_imm_support)) { + err = -ENOSYS; + break; + } + fw_flags |= FW_RI_RDMA_WRITE_WITH_IMMEDIATE; + /*FALLTHROUGH*/ case IB_WR_RDMA_WRITE: fw_opcode = FW_RI_RDMA_WRITE_WR; swsqe->opcode = FW_RI_RDMA_WRITE; @@ -1832,6 +1842,8 @@ c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, goto err6; } uresp.flags = 0; + if (rhp->rdev.adap->params.write_w_imm_support) + uresp.flags |= C4IW_QPF_WRITE_W_IMM; uresp.qid_mask = rhp->rdev.qpmask; uresp.sqid = qhp->wq.sq.qid; uresp.sq_size = qhp->wq.sq.size; diff --git a/sys/dev/cxgbe/iw_cxgbe/t4.h b/sys/dev/cxgbe/iw_cxgbe/t4.h index cf32c8cc3a65..1401883a21a7 100644 --- a/sys/dev/cxgbe/iw_cxgbe/t4.h +++ b/sys/dev/cxgbe/iw_cxgbe/t4.h @@ -205,8 +205,13 @@ struct t4_cqe { __be32 wrid_low; } gen; u64 drain_cookie; + struct { + __be32 mo; + __be32 msn; + __u64 imm_data; + } imm_data_rcqe; } u; - __be64 reserved[4]; + __be64 reserved[3]; __be64 bits_type_ts; }; @@ -256,6 +261,7 @@ struct t4_cqe { /* used for RQ completion processing */ #define CQE_WRID_STAG(x) (be32_to_cpu((x)->u.rcqe.stag)) #define CQE_WRID_MSN(x) (be32_to_cpu((x)->u.rcqe.msn)) +#define CQE_IMM_DATA(x) ((x)->u.imm_data_rcqe.imm_data) /* used for SQ completion processing */ #define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx) diff --git a/sys/dev/cxgbe/iw_cxgbe/user.h b/sys/dev/cxgbe/iw_cxgbe/user.h index 8408f3b6907b..ff1d86e56695 100644 --- a/sys/dev/cxgbe/iw_cxgbe/user.h +++ b/sys/dev/cxgbe/iw_cxgbe/user.h @@ -57,6 +57,10 @@ struct c4iw_create_cq_resp { __u32 reserved; /* explicit padding (optional for i386) */ }; +enum { + C4IW_QPF_WRITE_W_IMM = (1<<1) +}; + struct c4iw_create_qp_resp { __u64 ma_sync_key; __u64 sq_key;
