The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=99d683a32f5122465215e47479b9bedab0b0db8b
commit 99d683a32f5122465215e47479b9bedab0b0db8b Author: Vag Singh <[email protected]> AuthorDate: 2025-08-14 06:31:33 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2026-09-16 14:12:08 +0000 verbs/mlx5: Annotate ibv_wc and ibv_send_wr with endian PATCH 16/31] FreeBSD OFED support for DPDK MLX5 PMD a) verbs: Annotate ibv_wc helpers with endian This follows the scheme as used in the wc by introducing a ibv_wc_read_invalidated_rkey to access the host endian invalidated_rkey value with proper annotations. This is just an inline wrapper to allow sparse to work sensibly, not really a good reason to add another driver entry point. b) verbs: Annotate ibv_send_wr with endian This follows the scheme as used in the wc by introducing a transparent union in ibv_send_wr to indicate the invalidate_rkey is in host endian. c) mlx5: Add sparse annotations. Differential revision: https://reviews.freebsd.org/D32190 MFC after: 1 month --- contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 | 5 +- contrib/ofed/libibverbs/man/ibv_post_send.3 | 7 ++- contrib/ofed/libibverbs/verbs.h | 20 ++++++-- contrib/ofed/libmlx5/cq.c | 26 ++++++---- contrib/ofed/libmlx5/dbrec.c | 6 +-- contrib/ofed/libmlx5/mlx5.h | 14 +++--- contrib/ofed/libmlx5/mlx5dv.h | 68 +++++++++++++------------- contrib/ofed/libmlx5/qp.c | 10 ++-- contrib/ofed/libmlx5/verbs.c | 4 +- contrib/ofed/libmlx5/wqe.h | 34 ++++++------- 10 files changed, 111 insertions(+), 83 deletions(-) diff --git a/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 b/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 index 7dfbef28d241..e943e0e266c5 100644 --- a/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 +++ b/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 @@ -104,9 +104,12 @@ Below members and functions are used in order to poll the current completion. Th .BI "uint32_t ibv_wc_read_byte_len(struct ibv_cq_ex " "*cq"); \c Get the vendor error from the current completion. -.BI "uint32_t ibv_wc_read_imm_data(struct ibv_cq_ex " "*cq"); \c +.BI "__be32 ibv_wc_read_imm_data(struct ibv_cq_ex " "*cq"); \c Get the immediate data field from the current completion. +.BI "uint32_t ibv_wc_read_invalidated_rkey(struct ibv_cq_ex " "*cq"); \c + Get the rkey invalided by the SEND_INVAL from the current completion. + .BI "uint32_t ibv_wc_read_qp_num(struct ibv_cq_ex " "*cq"); \c Get the QP number field from the current completion. diff --git a/contrib/ofed/libibverbs/man/ibv_post_send.3 b/contrib/ofed/libibverbs/man/ibv_post_send.3 index ebbbb806882e..d0ee275329d3 100644 --- a/contrib/ofed/libibverbs/man/ibv_post_send.3 +++ b/contrib/ofed/libibverbs/man/ibv_post_send.3 @@ -35,7 +35,12 @@ struct ibv_sge *sg_list; /* Pointer to the s/g array */ int num_sge; /* Size of the s/g array */ enum ibv_wr_opcode opcode; /* Operation type */ int send_flags; /* Flags of the WR properties */ -uint32_t imm_data; /* Immediate data (in network byte order) */ +union { +.in +8 +__be32 imm_data; /* Immediate data (in network byte order) */ +uint32_t invalidate_rkey; /* Remote rkey to invalidate */ +.in -8 +}; union { .in +8 struct { diff --git a/contrib/ofed/libibverbs/verbs.h b/contrib/ofed/libibverbs/verbs.h index a3eb1826a09a..5ef6430dca93 100644 --- a/contrib/ofed/libibverbs/verbs.h +++ b/contrib/ofed/libibverbs/verbs.h @@ -947,9 +947,12 @@ struct ibv_send_wr { int num_sge; enum ibv_wr_opcode opcode; int send_flags; + /* When opcode is *_WITH_IMM: Immediate data in network byte order. + * When opcode is *_INV: Stores the rkey to invalidate + */ union { - __be32 imm_data; - u32 invalidate_rkey; + __be32 imm_data; + uint32_t invalidate_rkey; }; union { struct { @@ -1102,7 +1105,7 @@ struct ibv_cq_ex { enum ibv_wc_opcode (*read_opcode)(struct ibv_cq_ex *current); uint32_t (*read_vendor_err)(struct ibv_cq_ex *current); uint32_t (*read_byte_len)(struct ibv_cq_ex *current); - uint32_t (*read_imm_data)(struct ibv_cq_ex *current); + __be32 (*read_imm_data)(struct ibv_cq_ex *current); uint32_t (*read_qp_num)(struct ibv_cq_ex *current); uint32_t (*read_src_qp)(struct ibv_cq_ex *current); int (*read_wc_flags)(struct ibv_cq_ex *current); @@ -1150,9 +1153,18 @@ static inline uint32_t ibv_wc_read_byte_len(struct ibv_cq_ex *cq) return cq->read_byte_len(cq); } -static inline uint32_t ibv_wc_read_imm_data(struct ibv_cq_ex *cq) +static inline __be32 ibv_wc_read_imm_data(struct ibv_cq_ex *cq) +{ + return cq->read_imm_data(cq); +} + +static inline uint32_t ibv_wc_read_invalidated_rkey(struct ibv_cq_ex *cq) { +#ifdef __CHECKER__ + return (__attribute__((force)) uint32_t)cq->read_imm_data(cq); +#else return cq->read_imm_data(cq); +#endif } static inline uint32_t ibv_wc_read_qp_num(struct ibv_cq_ex *cq) diff --git a/contrib/ofed/libmlx5/cq.c b/contrib/ofed/libmlx5/cq.c index d9fe5c075ad4..a23cd4416f24 100644 --- a/contrib/ofed/libmlx5/cq.c +++ b/contrib/ofed/libmlx5/cq.c @@ -181,6 +181,15 @@ static inline int handle_responder_lazy(struct mlx5_cq *cq, struct mlx5_cqe64 *c return err; } +/* Returns IBV_WC_IP_CSUM_OK or 0 */ +static inline int get_csum_ok(struct mlx5_cqe64 *cqe) +{ + return (((cqe->hds_ip_ext & (MLX5_CQE_L4_OK | MLX5_CQE_L3_OK)) == + (MLX5_CQE_L4_OK | MLX5_CQE_L3_OK)) & + (get_cqe_l3_hdr_type(cqe) == MLX5_CQE_L3_HDR_TYPE_IPV4)) + << IBV_WC_IP_CSUM_OK_SHIFT; +} + static inline int handle_responder(struct ibv_wc *wc, struct mlx5_cqe64 *cqe, struct mlx5_resource *cur_rsc, struct mlx5_srq *srq) { @@ -205,11 +214,7 @@ static inline int handle_responder(struct ibv_wc *wc, struct mlx5_cqe64 *cqe, if (likely(cur_rsc->type == MLX5_RSC_TYPE_QP)) { wq = &qp->rq; if (qp->qp_cap_cache & MLX5_RX_CSUM_VALID) - wc->wc_flags |= (!!(cqe->hds_ip_ext & MLX5_CQE_L4_OK) & - !!(cqe->hds_ip_ext & MLX5_CQE_L3_OK) & - (get_cqe_l3_hdr_type(cqe) == - MLX5_CQE_L3_HDR_TYPE_IPV4)) << - IBV_WC_IP_CSUM_OK_SHIFT; + wc->wc_flags |= get_csum_ok(cqe); } else { wq = &(rsc_to_mrwq(cur_rsc)->rq); } @@ -244,7 +249,7 @@ static inline int handle_responder(struct ibv_wc *wc, struct mlx5_cqe64 *cqe, case MLX5_CQE_RESP_SEND_INV: wc->opcode = IBV_WC_RECV; wc->wc_flags |= IBV_WC_WITH_INV; - wc->imm_data = be32toh(cqe->imm_inval_pkey); + wc->invalidated_rkey = be32toh(cqe->imm_inval_pkey); break; } wc->slid = be16toh(cqe->slid); @@ -260,7 +265,7 @@ static inline int handle_responder(struct ibv_wc *wc, struct mlx5_cqe64 *cqe, static void dump_cqe(FILE *fp, void *buf) { - uint32_t *p = buf; + __be32 *p = buf; int i; for (i = 0; i < 16; i += 4) @@ -1139,13 +1144,16 @@ static inline uint32_t mlx5_cq_read_wc_vendor_err(struct ibv_cq_ex *ibcq) return ecqe->vendor_err_synd; } -static inline uint32_t mlx5_cq_read_wc_imm_data(struct ibv_cq_ex *ibcq) +static inline __be32 mlx5_cq_read_wc_imm_data(struct ibv_cq_ex *ibcq) { struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); switch (mlx5dv_get_cqe_opcode(cq->cqe64)) { case MLX5_CQE_RESP_SEND_INV: - return be32toh(cq->cqe64->imm_inval_pkey); + /* This is returning invalidate_rkey which is in host order, + * see ibv_wc_read_invalidated_rkey + */ + return (__be32)be32toh(cq->cqe64->imm_inval_pkey); default: return cq->cqe64->imm_inval_pkey; } diff --git a/contrib/ofed/libmlx5/dbrec.c b/contrib/ofed/libmlx5/dbrec.c index f346939a8edf..20e37cac8cf6 100644 --- a/contrib/ofed/libmlx5/dbrec.c +++ b/contrib/ofed/libmlx5/dbrec.c @@ -80,10 +80,10 @@ static struct mlx5_db_page *__add_page(struct mlx5_context *context) return page; } -uint32_t *mlx5_alloc_dbrec(struct mlx5_context *context) +__be32 *mlx5_alloc_dbrec(struct mlx5_context *context) { struct mlx5_db_page *page; - uint32_t *db = NULL; + __be32 *db = NULL; int i, j; pthread_mutex_lock(&context->db_list_mutex); @@ -113,7 +113,7 @@ out: return db; } -void mlx5_free_db(struct mlx5_context *context, uint32_t *db) +void mlx5_free_db(struct mlx5_context *context, __be32 *db) { struct mlx5_db_page *page; uintptr_t ps = to_mdev(context->ibv_ctx.device)->page_size; diff --git a/contrib/ofed/libmlx5/mlx5.h b/contrib/ofed/libmlx5/mlx5.h index 74c6685a1e73..dbd17db62ffd 100644 --- a/contrib/ofed/libmlx5/mlx5.h +++ b/contrib/ofed/libmlx5/mlx5.h @@ -325,7 +325,7 @@ struct mlx5_cq { struct mlx5_spinlock lock; uint32_t cqn; uint32_t cons_index; - uint32_t *dbrec; + __be32 *dbrec; int arm_sn; int cqe_sz; int resize_cqe_sz; @@ -353,7 +353,7 @@ struct mlx5_srq { int wqe_shift; int head; int tail; - uint32_t *db; + __be32 *db; uint16_t counter; int wq_sig; }; @@ -412,7 +412,7 @@ struct mlx5_qp { uint8_t sq_signal_bits; struct mlx5_wq sq; - uint32_t *db; + __be32 *db; struct mlx5_wq rq; int wq_sig; uint32_t qp_cap_cache; @@ -434,9 +434,9 @@ struct mlx5_rwq { struct mlx5_buf buf; int buf_size; struct mlx5_wq rq; - uint32_t *db; + __be32 *db; void *pbuff; - uint32_t *recv_db; + __be32 *recv_db; int wq_sig; }; @@ -563,8 +563,8 @@ void mlx5_get_alloc_type(const char *component, enum mlx5_alloc_type default_alloc_type); int mlx5_use_huge(const char *key); -uint32_t *mlx5_alloc_dbrec(struct mlx5_context *context); -void mlx5_free_db(struct mlx5_context *context, uint32_t *db); +__be32 *mlx5_alloc_dbrec(struct mlx5_context *context); +void mlx5_free_db(struct mlx5_context *context, __be32 *db); int mlx5_query_device(struct ibv_context *context, struct ibv_device_attr *attr); diff --git a/contrib/ofed/libmlx5/mlx5dv.h b/contrib/ofed/libmlx5/mlx5dv.h index 93aaa4bb395c..856b6c68558d 100644 --- a/contrib/ofed/libmlx5/mlx5dv.h +++ b/contrib/ofed/libmlx5/mlx5dv.h @@ -112,7 +112,7 @@ enum mlx5dv_qp_comp_mask { }; struct mlx5dv_qp { - uint32_t *dbrec; + __be32 *dbrec; struct { void *buf; uint32_t wqe_cnt; @@ -133,7 +133,7 @@ struct mlx5dv_qp { struct mlx5dv_cq { void *buf; - uint32_t *dbrec; + __be32 *dbrec; uint32_t cqe_cnt; uint32_t cqe_size; void *uar; @@ -143,7 +143,7 @@ struct mlx5dv_cq { struct mlx5dv_srq { void *buf; - uint32_t *dbrec; + __be32 *dbrec; uint32_t stride; uint32_t head; uint32_t tail; @@ -152,7 +152,7 @@ struct mlx5dv_srq { struct mlx5dv_rwq { void *buf; - uint32_t *dbrec; + __be32 *dbrec; uint32_t wqe_cnt; uint32_t stride; uint64_t comp_mask; @@ -292,18 +292,18 @@ struct mlx5_cqe64 { uint8_t rsvd0[17]; uint8_t ml_path; uint8_t rsvd20[4]; - uint16_t slid; - uint32_t flags_rqpn; + __be16 slid; + __be32 flags_rqpn; uint8_t hds_ip_ext; uint8_t l4_hdr_type_etc; - uint16_t vlan_info; - uint32_t srqn_uidx; - uint32_t imm_inval_pkey; + __be16 vlan_info; + __be32 srqn_uidx; + __be32 imm_inval_pkey; uint8_t rsvd40[4]; - uint32_t byte_cnt; + __be32 byte_cnt; __be64 timestamp; - uint32_t sop_drop_qpn; - uint16_t wqe_counter; + __be32 sop_drop_qpn; + __be16 wqe_counter; uint8_t signature; uint8_t op_own; }; @@ -379,43 +379,43 @@ enum { struct mlx5_wqe_srq_next_seg { uint8_t rsvd0[2]; - uint16_t next_wqe_index; + __be16 next_wqe_index; uint8_t signature; uint8_t rsvd1[11]; }; struct mlx5_wqe_data_seg { - uint32_t byte_count; - uint32_t lkey; - uint64_t addr; + __be32 byte_count; + __be32 lkey; + __be64 addr; }; struct mlx5_wqe_ctrl_seg { - uint32_t opmod_idx_opcode; - uint32_t qpn_ds; + __be32 opmod_idx_opcode; + __be32 qpn_ds; uint8_t signature; uint8_t rsvd[2]; uint8_t fm_ce_se; - uint32_t imm; + __be32 imm; }; struct mlx5_wqe_av { union { struct { - uint32_t qkey; - uint32_t reserved; + __be32 qkey; + __be32 reserved; } qkey; - uint64_t dc_key; + __be64 dc_key; } key; - uint32_t dqp_dct; + __be32 dqp_dct; uint8_t stat_rate_sl; uint8_t fl_mlid; - uint16_t rlid; + __be16 rlid; uint8_t reserved0[4]; uint8_t rmac[6]; uint8_t tclass; uint8_t hop_limit; - uint32_t grh_gid_fl; + __be32 grh_gid_fl; uint8_t rgid[16]; }; @@ -424,14 +424,14 @@ struct mlx5_wqe_datagram_seg { }; struct mlx5_wqe_raddr_seg { - uint64_t raddr; - uint32_t rkey; - uint32_t reserved; + __be64 raddr; + __be32 rkey; + __be32 reserved; }; struct mlx5_wqe_atomic_seg { - uint64_t swap_add; - uint64_t compare; + __be64 swap_add; + __be64 compare; }; struct mlx5_wqe_inl_data_seg { @@ -439,12 +439,12 @@ struct mlx5_wqe_inl_data_seg { }; struct mlx5_wqe_eth_seg { - uint32_t rsvd0; + __be32 rsvd0; uint8_t cs_flags; uint8_t rsvd1; - uint16_t mss; - uint32_t rsvd2; - uint16_t inline_hdr_sz; + __be16 mss; + __be32 rsvd2; + __be16 inline_hdr_sz; uint8_t inline_hdr_start[2]; uint8_t inline_hdr[16]; }; diff --git a/contrib/ofed/libmlx5/qp.c b/contrib/ofed/libmlx5/qp.c index 8c4ab9507cc8..ddd7231b4d3e 100644 --- a/contrib/ofed/libmlx5/qp.c +++ b/contrib/ofed/libmlx5/qp.c @@ -253,14 +253,14 @@ static void mlx5_bf_copy(unsigned long long *dst, unsigned long long *src, } } -static uint32_t send_ieth(struct ibv_send_wr *wr) +static __be32 send_ieth(struct ibv_send_wr *wr) { switch (wr->opcode) { case IBV_WR_SEND_WITH_IMM: case IBV_WR_RDMA_WRITE_WITH_IMM: return wr->imm_data; case IBV_WR_SEND_WITH_INV: - return htobe32(wr->imm_data); + return htobe32(wr->invalidate_rkey); default: return 0; } @@ -413,7 +413,7 @@ static inline int copy_eth_inline_headers(struct ibv_qp *ibqp, #undef ALIGN #define ALIGN(x, log_a) ((((x) + (1 << (log_a)) - 1)) & ~((1 << (log_a)) - 1)) -static inline uint16_t get_klm_octo(int nentries) +static inline __be16 get_klm_octo(int nentries) { return htobe16(ALIGN(nentries, 3) / 2); } @@ -742,7 +742,7 @@ static inline int _mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, struct ibv_mw_bind_info bind_info = {}; next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; - ctrl->imm = htobe32(wr->imm_data); + ctrl->imm = htobe32(wr->invalidate_rkey); err = set_bind_wr(qp, IBV_MW_TYPE_2, 0, &bind_info, ibqp->qp_num, &seg, &size); @@ -787,7 +787,7 @@ static inline int _mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, struct ibv_mw_bind_info bind_info = {}; next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; - ctrl->imm = htobe32(wr->imm_data); + ctrl->imm = htobe32(wr->invalidate_rkey); err = set_bind_wr(qp, IBV_MW_TYPE_2, 0, &bind_info, ibqp->qp_num, &seg, &size); diff --git a/contrib/ofed/libmlx5/verbs.c b/contrib/ofed/libmlx5/verbs.c index 6c296bb69f6f..5d97758738af 100644 --- a/contrib/ofed/libmlx5/verbs.c +++ b/contrib/ofed/libmlx5/verbs.c @@ -1579,7 +1579,7 @@ int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, struct mlx5_qp *mqp = to_mqp(qp); struct mlx5_context *context = to_mctx(qp->context); int ret; - uint32_t *db; + __be32 *db; if (mqp->rss_qp) return ENOSYS; @@ -1662,7 +1662,7 @@ struct ibv_ah *mlx5_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) struct ibv_port_attr port_attr; struct mlx5_ah *ah; uint32_t gid_type; - uint32_t tmp; + __be32 tmp; uint8_t grh; int is_eth; diff --git a/contrib/ofed/libmlx5/wqe.h b/contrib/ofed/libmlx5/wqe.h index cbe05d3d6738..063dc9ab9a03 100644 --- a/contrib/ofed/libmlx5/wqe.h +++ b/contrib/ofed/libmlx5/wqe.h @@ -51,7 +51,7 @@ struct mlx5_eqe_qp_srq { }; struct mlx5_wqe_xrc_seg { - uint32_t xrc_srqn; + __be32 xrc_srqn; uint8_t rsvd[12]; }; @@ -89,17 +89,17 @@ enum { struct mlx5_wqe_umr_ctrl_seg { uint8_t flags; uint8_t rsvd0[3]; - uint16_t klm_octowords; - uint16_t translation_offset; - uint64_t mkey_mask; + __be16 klm_octowords; + __be16 translation_offset; + __be64 mkey_mask; uint8_t rsvd1[32]; }; struct mlx5_wqe_umr_klm_seg { /* up to 2GB */ - uint32_t byte_count; - uint32_t mkey; - uint64_t address; + __be32 byte_count; + __be32 mkey; + __be64 address; }; union mlx5_wqe_umr_inline_seg { @@ -123,17 +123,17 @@ struct mlx5_wqe_mkey_context_seg { uint8_t reserved1; uint8_t access_flags; uint8_t sf; - uint32_t qpn_mkey; - uint32_t reserved2; - uint32_t flags_pd; - uint64_t start_addr; - uint64_t len; - uint32_t bsf_octword_size; - uint32_t reserved3[4]; - uint32_t translations_octword_size; + __be32 qpn_mkey; + __be32 reserved2; + __be32 flags_pd; + __be64 start_addr; + __be64 len; + __be32 bsf_octword_size; + __be32 reserved3[4]; + __be32 translations_octword_size; uint8_t reserved4[3]; uint8_t log_page_size; - uint32_t reserved; + __be32 reserved; union mlx5_wqe_umr_inline_seg inseg[0]; }; @@ -183,7 +183,7 @@ struct mlx5_wqe_signature_seg { }; struct mlx5_wqe_inline_seg { - uint32_t byte_count; + __be32 byte_count; };
