From: Haggai Abramonvsky <[email protected]> When working with CQE version 1, the CQE format is different, therefore we need a new poll_cq function that is compatible with the new CQE format.
When allocating the user-context, we decide on the CQE version, and according to this, we choose which poll_cq function to use; the old one that supports the old CQE format or the new one. Change-Id: I8c3174b18ea8e5857fd3a02333f5a8495b25c878 Signed-off-by: Haggai Abramovsky <[email protected]> --- src/cq.c | 217 +++++++++++++++++++++++++++++++++++++++++++++------------ src/mlx5-abi.h | 3 +- src/mlx5.h | 12 ++++ 3 files changed, 187 insertions(+), 45 deletions(-) diff --git a/src/cq.c b/src/cq.c index 63c3ab0..a1fdac3 100644 --- a/src/cq.c +++ b/src/cq.c @@ -116,7 +116,7 @@ struct mlx5_cqe64 { uint16_t slid; uint32_t flags_rqpn; uint8_t rsvd28[4]; - uint32_t srqn; + uint32_t srqn_uidx; uint32_t imm_inval_pkey; uint8_t rsvd40[4]; uint32_t byte_cnt; @@ -362,21 +362,120 @@ static void mlx5_get_cycles(uint64_t *cycles) } #endif -static int mlx5_poll_one(struct mlx5_cq *cq, - struct mlx5_qp **cur_qp, +static inline struct mlx5_qp *get_req_context(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t rsn, int cqe_ver) + __attribute__((always_inline)); +static inline struct mlx5_qp *get_req_context(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t rsn, int cqe_ver) +{ + if (!*cur_rsc || (rsn != (*cur_rsc)->rsn)) + *cur_rsc = cqe_ver ? mlx5_find_uidx(mctx, rsn) : + (struct mlx5_resource *)mlx5_find_qp(mctx, rsn); + + return rsc_to_mqp(*cur_rsc); +} + +static inline int get_resp_cxt_v1(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + struct mlx5_srq **cur_srq, + uint32_t uidx, int *is_srq) + __attribute__((always_inline)); +static inline int get_resp_cxt_v1(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + struct mlx5_srq **cur_srq, + uint32_t uidx, int *is_srq) +{ + struct mlx5_qp *mqp; + + if (!*cur_rsc || (uidx != (*cur_rsc)->rsn)) { + *cur_rsc = mlx5_find_uidx(mctx, uidx); + if (unlikely(!*cur_rsc)) + return CQ_POLL_ERR; + } + + switch ((*cur_rsc)->type) { + case MLX5_RSC_TYPE_QP: + mqp = rsc_to_mqp(*cur_rsc); + if (mqp->verbs_qp.qp.srq) { + *cur_srq = to_msrq(mqp->verbs_qp.qp.srq); + *is_srq = 1; + } + break; + case MLX5_RSC_TYPE_XSRQ: + *cur_srq = rsc_to_msrq(*cur_rsc); + *is_srq = 1; + break; + default: + return CQ_POLL_ERR; + } + + return CQ_OK; +} + +static inline int get_resp_ctx(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t qpn) + __attribute__((always_inline)); +static inline int get_resp_ctx(struct mlx5_context *mctx, + struct mlx5_resource **cur_rsc, + uint32_t qpn) +{ + if (!*cur_rsc || (qpn != (*cur_rsc)->rsn)) { + /* + * We do not have to take the QP table lock here, + * because CQs will be locked while QPs are removed + * from the table. + */ + *cur_rsc = (struct mlx5_resource *)mlx5_find_qp(mctx, qpn); + if (unlikely(!*cur_rsc)) + return CQ_POLL_ERR; + } + + return CQ_OK; +} + +static inline int get_srq_ctx(struct mlx5_context *mctx, + struct mlx5_srq **cur_srq, + uint32_t srqn_uidx) + __attribute__((always_inline)); +static inline int get_srq_ctx(struct mlx5_context *mctx, + struct mlx5_srq **cur_srq, + uint32_t srqn) +{ + if (!*cur_srq || (srqn != (*cur_srq)->srqn)) { + *cur_srq = mlx5_find_srq(mctx, srqn); + if (unlikely(!*cur_srq)) + return CQ_POLL_ERR; + } + + return CQ_OK; +} + +static inline int mlx5_poll_one(struct mlx5_cq *cq, + struct mlx5_resource **cur_rsc, struct mlx5_srq **cur_srq, - struct ibv_wc *wc) + struct ibv_wc *wc, int cqe_ver) + __attribute__((always_inline)); +static inline int mlx5_poll_one(struct mlx5_cq *cq, + struct mlx5_resource **cur_rsc, + struct mlx5_srq **cur_srq, + struct ibv_wc *wc, int cqe_ver) { struct mlx5_cqe64 *cqe64; struct mlx5_wq *wq; uint16_t wqe_ctr; void *cqe; uint32_t qpn; - uint32_t srqn; + uint32_t srqn_uidx; int idx; uint8_t opcode; struct mlx5_err_cqe *ecqe; int err; + int is_srq = 0; + struct mlx5_qp *mqp = NULL; + struct mlx5_context *mctx = to_mctx(cq->ibv_cq.context); cqe = next_cqe_sw(cq); if (!cqe) @@ -384,6 +483,7 @@ static int mlx5_poll_one(struct mlx5_cq *cq, cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64; + opcode = cqe64->op_own >> 4; ++cq->cons_index; VALGRIND_MAKE_MEM_DEFINED(cqe64, sizeof *cqe64); @@ -396,51 +496,32 @@ static int mlx5_poll_one(struct mlx5_cq *cq, #ifdef MLX5_DEBUG if (mlx5_debug_mask & MLX5_DBG_CQ_CQE) { - FILE *fp = to_mctx(cq->ibv_cq.context)->dbg_fp; + FILE *fp = mctx->dbg_fp; mlx5_dbg(fp, MLX5_DBG_CQ_CQE, "dump cqe for cqn 0x%x:\n", cq->cqn); dump_cqe(fp, cqe64); } #endif - srqn = ntohl(cqe64->srqn) & 0xffffff; qpn = ntohl(cqe64->sop_drop_qpn) & 0xffffff; - if (srqn) { - if (!*cur_srq || (srqn != (*cur_srq)->srqn)) { - *cur_srq = mlx5_find_srq(to_mctx(cq->ibv_cq.context), - srqn); - if (unlikely(!*cur_srq)) - return CQ_POLL_ERR; - } - } else { - if (!*cur_qp || (qpn != (*cur_qp)->ibv_qp.qp_num)) { - /* - * We do not have to take the QP table lock here, - * because CQs will be locked while QPs are removed - * from the table. - */ - *cur_qp = mlx5_find_qp(to_mctx(cq->ibv_cq.context), - qpn); - if (unlikely(!*cur_qp)) - return CQ_POLL_ERR; - } - } - wc->wc_flags = 0; - wc->qp_num = qpn; - opcode = cqe64->op_own >> 4; switch (opcode) { case MLX5_CQE_REQ: - wq = &(*cur_qp)->sq; + mqp = get_req_context(mctx, cur_rsc, + (cqe_ver ? (ntohl(cqe64->srqn_uidx) & 0xffffff) : qpn), + cqe_ver); + if (unlikely(!mqp)) + return CQ_POLL_ERR; + wq = &mqp->sq; wqe_ctr = ntohs(cqe64->wqe_counter); idx = wqe_ctr & (wq->wqe_cnt - 1); handle_good_req(wc, cqe64); if (cqe64->op_own & MLX5_INLINE_SCATTER_32) - err = mlx5_copy_to_send_wqe(*cur_qp, wqe_ctr, cqe, + err = mlx5_copy_to_send_wqe(mqp, wqe_ctr, cqe, wc->byte_len); else if (cqe64->op_own & MLX5_INLINE_SCATTER_64) - err = mlx5_copy_to_send_wqe(*cur_qp, wqe_ctr, cqe - 1, + err = mlx5_copy_to_send_wqe(mqp, wqe_ctr, cqe - 1, wc->byte_len); else err = 0; @@ -453,20 +534,35 @@ static int mlx5_poll_one(struct mlx5_cq *cq, case MLX5_CQE_RESP_SEND: case MLX5_CQE_RESP_SEND_IMM: case MLX5_CQE_RESP_SEND_INV: - wc->status = handle_responder(wc, cqe64, *cur_qp, - srqn ? *cur_srq : NULL); + srqn_uidx = ntohl(cqe64->srqn_uidx) & 0xffffff; + if (cqe_ver) { + err = get_resp_cxt_v1(mctx, cur_rsc, cur_srq, srqn_uidx, &is_srq); + } else { + if (srqn_uidx) { + err = get_srq_ctx(mctx, cur_srq, srqn_uidx); + is_srq = 1; + } else { + err = get_resp_ctx(mctx, cur_rsc, qpn); + } + } + if (unlikely(err)) + return err; + + wc->status = handle_responder(wc, cqe64, rsc_to_mqp(*cur_rsc), + is_srq ? *cur_srq : NULL); break; case MLX5_CQE_RESIZE_CQ: break; case MLX5_CQE_REQ_ERR: case MLX5_CQE_RESP_ERR: + srqn_uidx = ntohl(cqe64->srqn_uidx) & 0xffffff; ecqe = (struct mlx5_err_cqe *)cqe64; mlx5_handle_error_cqe(ecqe, wc); if (unlikely(ecqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR && ecqe->syndrome != MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR)) { - FILE *fp = to_mctx(cq->ibv_cq.context)->dbg_fp; + FILE *fp = mctx->dbg_fp; fprintf(fp, PFX "%s: got completion with error:\n", - to_mctx(cq->ibv_cq.context)->hostname); + mctx->hostname); dump_cqe(fp, ecqe); if (mlx5_freeze_on_error_cqe) { fprintf(fp, PFX "freezing at poll cq..."); @@ -476,18 +572,35 @@ static int mlx5_poll_one(struct mlx5_cq *cq, } if (opcode == MLX5_CQE_REQ_ERR) { - wq = &(*cur_qp)->sq; + mqp = get_req_context(mctx, cur_rsc, (cqe_ver ? srqn_uidx : qpn), cqe_ver); + if (unlikely(!mqp)) + return CQ_POLL_ERR; + wq = &mqp->sq; wqe_ctr = ntohs(cqe64->wqe_counter); idx = wqe_ctr & (wq->wqe_cnt - 1); wc->wr_id = wq->wrid[idx]; wq->tail = wq->wqe_head[idx] + 1; } else { - if (*cur_srq) { + if (cqe_ver) { + err = get_resp_cxt_v1(mctx, cur_rsc, cur_srq, srqn_uidx, &is_srq); + } else { + if (srqn_uidx) { + err = get_srq_ctx(mctx, cur_srq, srqn_uidx); + is_srq = 1; + } else { + err = get_resp_ctx(mctx, cur_rsc, qpn); + } + } + if (unlikely(err)) + return CQ_POLL_ERR; + + if (is_srq) { wqe_ctr = ntohs(cqe64->wqe_counter); wc->wr_id = (*cur_srq)->wrid[wqe_ctr]; mlx5_free_srq_wqe(*cur_srq, wqe_ctr); } else { - wq = &(*cur_qp)->rq; + mqp = rsc_to_mqp(*cur_rsc); + wq = &mqp->rq; wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; ++wq->tail; } @@ -495,13 +608,19 @@ static int mlx5_poll_one(struct mlx5_cq *cq, break; } + wc->qp_num = qpn; + return CQ_OK; } -int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +static inline int poll_cq(struct ibv_cq *ibcq, int ne, + struct ibv_wc *wc, int cqe_ver) + __attribute__((always_inline)); +static inline int poll_cq(struct ibv_cq *ibcq, int ne, + struct ibv_wc *wc, int cqe_ver) { struct mlx5_cq *cq = to_mcq(ibcq); - struct mlx5_qp *qp = NULL; + struct mlx5_resource *rsc = NULL; struct mlx5_srq *srq = NULL; int npolled; int err = CQ_OK; @@ -519,7 +638,7 @@ int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) mlx5_spin_lock(&cq->lock); for (npolled = 0; npolled < ne; ++npolled) { - err = mlx5_poll_one(cq, &qp, &srq, wc + npolled); + err = mlx5_poll_one(cq, &rsc, &srq, wc + npolled, cqe_ver); if (err != CQ_OK) break; } @@ -551,6 +670,16 @@ int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) return err == CQ_POLL_ERR ? err : npolled; } +int mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +{ + return poll_cq(ibcq, ne, wc, 0); +} + +int mlx5_poll_cq_v1(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) +{ + return poll_cq(ibcq, ne, wc, 1); +} + int mlx5_arm_cq(struct ibv_cq *ibvcq, int solicited) { struct mlx5_cq *cq = to_mcq(ibvcq); @@ -622,7 +751,7 @@ void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t rsn, struct mlx5_srq *srq) cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe); cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64; if (is_equal_rsn(cqe64, rsn)) { - if (srq && (ntohl(cqe64->srqn) & 0xffffff)) + if (srq && (ntohl(cqe64->srqn_uidx) & 0xffffff)) mlx5_free_srq_wqe(srq, ntohs(cqe64->wqe_counter)); ++nfreed; } else if (nfreed) { diff --git a/src/mlx5-abi.h b/src/mlx5-abi.h index ba31615..d0a0825 100644 --- a/src/mlx5-abi.h +++ b/src/mlx5-abi.h @@ -70,7 +70,8 @@ struct mlx5_alloc_ucontext_resp { __u32 max_recv_wr; __u32 max_srq_recv_wr; __u16 num_ports; - __u16 reserved; + __u8 cqe_version; + __u8 reserved; }; struct mlx5_alloc_pd_resp { diff --git a/src/mlx5.h b/src/mlx5.h index dd618bd..8b360fe 100644 --- a/src/mlx5.h +++ b/src/mlx5.h @@ -306,6 +306,7 @@ struct mlx5_context { char hostname[40]; struct mlx5_spinlock hugetlb_lock; struct list_head hugetlb_list; + uint8_t cqe_version; }; struct mlx5_bitmap { @@ -544,6 +545,16 @@ static inline int max_int(int a, int b) return a > b ? a : b; } +static inline struct mlx5_qp *rsc_to_mqp(struct mlx5_resource *rsc) +{ + return (struct mlx5_qp *)rsc; +} + +static inline struct mlx5_srq *rsc_to_msrq(struct mlx5_resource *rsc) +{ + return (struct mlx5_srq *)rsc; +} + int mlx5_alloc_buf(struct mlx5_buf *buf, size_t size, int page_size); void mlx5_free_buf(struct mlx5_buf *buf); int mlx5_alloc_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf, @@ -590,6 +601,7 @@ int mlx5_free_cq_buf(struct mlx5_context *ctx, struct mlx5_buf *buf); int mlx5_resize_cq(struct ibv_cq *cq, int cqe); int mlx5_destroy_cq(struct ibv_cq *cq); int mlx5_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); +int mlx5_poll_cq_v1(struct ibv_cq *cq, int ne, struct ibv_wc *wc); int mlx5_arm_cq(struct ibv_cq *cq, int solicited); void mlx5_cq_event(struct ibv_cq *cq); void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq); -- 1.8.3.1 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
