The branch stable/13 has been updated by erj: URL: https://cgit.FreeBSD.org/src/commit/?id=f5f2cc9ee688b0f6fc372f784ae8642de2e6ef7e
commit f5f2cc9ee688b0f6fc372f784ae8642de2e6ef7e Author: Bartosz Sobczak <[email protected]> AuthorDate: 2023-07-05 14:23:06 +0000 Commit: Eric Joyner <[email protected]> CommitDate: 2023-09-22 21:54:37 +0000 irdma(4): upgrade to 1.2.17-k Update Intel irdma driver to version 1.2.17-k Notable changes: - pf-reset handling improvements, including fixes in communication with if_ice(4) - avoid racing when handling various events - adding sw stats sysctls - hand over pe_criterr handling from ice(4) to irdma(4) - debug prints adjustments - fix crash after changes in irdma_add_mqh_ifa_cb Additional fixes in stable/13 but not in original cherry-picked commit: - code appearance improvements - bug fixes in fbsd_kcompat.c - avoid waiting procedures under lock in irdma_add_mqh_ifa_cb - busy wait for manage_qhash - stats don't need to be RDTUN, RD is enough Signed-off-by: Bartosz Sobczak <[email protected]> Signed-off-by: Eric Joyner <[email protected]> Reviewed by: erj@ Sponsored by: Intel Corporation Differential Revision: https://reviews.freebsd.org/D41425 (cherry picked from commit 01fbb86991d524288a785b544c29d9c5ea1b61fb) --- contrib/ofed/libirdma/abi.h | 1 - contrib/ofed/libirdma/i40e_devids.h | 1 - contrib/ofed/libirdma/i40iw_hw.h | 3 +- contrib/ofed/libirdma/ice_devids.h | 1 - contrib/ofed/libirdma/irdma-abi.h | 1 - contrib/ofed/libirdma/irdma.h | 1 - contrib/ofed/libirdma/irdma_defs.h | 26 +- contrib/ofed/libirdma/irdma_uk.c | 61 +--- contrib/ofed/libirdma/irdma_umain.c | 3 +- contrib/ofed/libirdma/irdma_umain.h | 1 - contrib/ofed/libirdma/irdma_uquery.h | 1 - contrib/ofed/libirdma/irdma_user.h | 2 - contrib/ofed/libirdma/irdma_uverbs.c | 62 +++-- contrib/ofed/libirdma/osdep.h | 80 ++---- sys/dev/irdma/fbsd_kcompat.c | 444 +++++++++++++++++++++++++---- sys/dev/irdma/fbsd_kcompat.h | 37 ++- sys/dev/irdma/icrdma.c | 310 ++++++++++++++------- sys/dev/irdma/icrdma_hw.c | 2 +- sys/dev/irdma/irdma.h | 2 +- sys/dev/irdma/irdma_cm.c | 522 ++++++++++++++++------------------- sys/dev/irdma/irdma_cm.h | 13 +- sys/dev/irdma/irdma_ctrl.c | 164 ++++------- sys/dev/irdma/irdma_defs.h | 87 +++--- sys/dev/irdma/irdma_hmc.c | 6 +- sys/dev/irdma/irdma_hmc.h | 6 +- sys/dev/irdma/irdma_hw.c | 106 +++---- sys/dev/irdma/irdma_kcompat.c | 118 ++++---- sys/dev/irdma/irdma_main.h | 14 +- sys/dev/irdma/irdma_pble.c | 16 +- sys/dev/irdma/irdma_pble.h | 4 +- sys/dev/irdma/irdma_protos.h | 6 +- sys/dev/irdma/irdma_puda.c | 9 + sys/dev/irdma/irdma_puda.h | 4 +- sys/dev/irdma/irdma_type.h | 25 +- sys/dev/irdma/irdma_uda.c | 15 +- sys/dev/irdma/irdma_uda.h | 2 +- sys/dev/irdma/irdma_uk.c | 85 +----- sys/dev/irdma/irdma_user.h | 1 - sys/dev/irdma/irdma_utils.c | 229 +++++++++++---- sys/dev/irdma/irdma_verbs.c | 438 +++++++++++++++-------------- sys/dev/irdma/irdma_verbs.h | 92 +++++- sys/dev/irdma/irdma_ws.c | 4 +- sys/dev/irdma/osdep.h | 60 ++-- sys/modules/irdma/Makefile | 2 +- 44 files changed, 1761 insertions(+), 1306 deletions(-) diff --git a/contrib/ofed/libirdma/abi.h b/contrib/ofed/libirdma/abi.h index 6553ebcbcaca..1d84fbc747f6 100644 --- a/contrib/ofed/libirdma/abi.h +++ b/contrib/ofed/libirdma/abi.h @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef PROVIDER_IRDMA_ABI_H #define PROVIDER_IRDMA_ABI_H diff --git a/contrib/ofed/libirdma/i40e_devids.h b/contrib/ofed/libirdma/i40e_devids.h index 1b0eaae95b82..e775a75bade5 100644 --- a/contrib/ofed/libirdma/i40e_devids.h +++ b/contrib/ofed/libirdma/i40e_devids.h @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef I40E_DEVIDS_H #define I40E_DEVIDS_H diff --git a/contrib/ofed/libirdma/i40iw_hw.h b/contrib/ofed/libirdma/i40iw_hw.h index d04c37d689cb..fcbfea8dfe09 100644 --- a/contrib/ofed/libirdma/i40iw_hw.h +++ b/contrib/ofed/libirdma/i40iw_hw.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef I40IW_HW_H #define I40IW_HW_H diff --git a/contrib/ofed/libirdma/ice_devids.h b/contrib/ofed/libirdma/ice_devids.h index 57f26bc33260..57a7f2f7c2af 100644 --- a/contrib/ofed/libirdma/ice_devids.h +++ b/contrib/ofed/libirdma/ice_devids.h @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef ICE_DEVIDS_H #define ICE_DEVIDS_H diff --git a/contrib/ofed/libirdma/irdma-abi.h b/contrib/ofed/libirdma/irdma-abi.h index b7d4b61c162d..8c04dcce2e8b 100644 --- a/contrib/ofed/libirdma/irdma-abi.h +++ b/contrib/ofed/libirdma/irdma-abi.h @@ -35,7 +35,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef IRDMA_ABI_H #define IRDMA_ABI_H diff --git a/contrib/ofed/libirdma/irdma.h b/contrib/ofed/libirdma/irdma.h index 1dd09c36c7ea..0e2a5189acd1 100644 --- a/contrib/ofed/libirdma/irdma.h +++ b/contrib/ofed/libirdma/irdma.h @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef IRDMA_H #define IRDMA_H diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h index 3d8b59c4b78e..da9eea472568 100644 --- a/contrib/ofed/libirdma/irdma_defs.h +++ b/contrib/ofed/libirdma/irdma_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef IRDMA_DEFS_H #define IRDMA_DEFS_H @@ -312,7 +311,7 @@ #define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \ { \ - register __u32 offset; \ + __u32 offset; \ offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \ (_cqe) = (_cq)->cq_base[offset].buf; \ } @@ -338,7 +337,7 @@ #define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -349,7 +348,7 @@ } #define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -360,7 +359,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -371,7 +370,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -457,6 +456,19 @@ enum irdma_qp_wqe_size { IRDMA_WQE_SIZE_256 = 256, }; +enum irdma_ws_op_type { + IRDMA_WS_OP_TYPE_NODE = 0, + IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP, +}; + +enum irdma_ws_rate_limit_flags { + IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1, + IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2, + IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4, + IRDMA_WS_TREE_RATE_LIMITING = 0x8, + IRDMA_WS_PACING_CONTROL = 0x10, +}; + /** * set_64bit_val - set 64 bit value to hw wqe * @wqe_words: wqe addr to write diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c index 97e3ac553c26..5fa9d792745f 100644 --- a/contrib/ofed/libirdma/irdma_uk.c +++ b/contrib/ofed/libirdma/irdma_uk.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #include "osdep.h" #include "irdma_defs.h" @@ -275,7 +274,8 @@ irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 && (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; - wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity)); + wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, + qp->swqe_polarity ? 0 : 1)); } qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; @@ -683,8 +683,8 @@ irdma_set_mw_bind_wqe(__le64 * wqe, * @polarity: polarity of wqe valid bit */ static void -irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges, - u8 polarity) +irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, + u32 num_sges, u8 polarity) { u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; u32 quanta_bytes_remaining = 8; @@ -1173,7 +1173,7 @@ irdma_repost_rq_wqes(struct irdma_qp_uk *qp, u32 start_idx, u32 end_idx) { __le64 *dst_wqe, *src_wqe; - u32 wqe_idx; + u32 wqe_idx = 0; u8 wqe_quanta = qp->rq_wqe_size_multiplier; bool flip_polarity; u64 val; @@ -1480,7 +1480,8 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, sw_wqe = qp->sq_base[tail].elem; get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword); - info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); + info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, + wqe_qword); IRDMA_RING_SET_TAIL(qp->sq_ring, tail + qp->sq_wrtrk_array[tail].quanta); if (info->op_type != IRDMAQP_OP_NOP) { @@ -1690,6 +1691,7 @@ irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo, { bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false; int status; + irdma_get_wqe_shift(ukinfo->uk_attrs, imm_support ? ukinfo->max_sq_frag_cnt + 1 : ukinfo->max_sq_frag_cnt, @@ -1834,6 +1836,9 @@ irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq) if (polarity != temp) break; + /* Ensure CQE contents are read after valid bit is checked */ + udma_from_device_barrier(); + get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); if ((void *)(irdma_uintptr) comp_ctx == q) set_64bit_val(cqe, IRDMA_BYTE_8, 0); @@ -1845,48 +1850,6 @@ irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq) return 0; } -/** - * irdma_nop - post a nop - * @qp: hw qp ptr - * @wr_id: work request id - * @signaled: signaled for completion - * @post_sq: ring doorbell - */ -int -irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq) -{ - __le64 *wqe; - u64 hdr; - u32 wqe_idx; - struct irdma_post_sq_info info = {0}; - u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; - - info.push_wqe = qp->push_db ? true : false; - info.wr_id = wr_id; - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info); - if (!wqe) - return ENOSPC; - - set_64bit_val(wqe, IRDMA_BYTE_0, 0); - set_64bit_val(wqe, IRDMA_BYTE_8, 0); - set_64bit_val(wqe, IRDMA_BYTE_16, 0); - - hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | - FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | - FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); - - udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - - if (info.push_wqe) - irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - else if (post_sq) - irdma_uk_qp_post_wr(qp); - - return 0; -} - /** * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ * @frag_cnt: number of fragments diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c index 6c823646b375..2902dd963e18 100644 --- a/contrib/ofed/libirdma/irdma_umain.c +++ b/contrib/ofed/libirdma/irdma_umain.c @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #include <sys/mman.h> @@ -49,7 +48,7 @@ /** * Driver version */ -char libirdma_version[] = "1.1.11-k"; +char libirdma_version[] = "1.2.17-k"; unsigned int irdma_dbg; diff --git a/contrib/ofed/libirdma/irdma_umain.h b/contrib/ofed/libirdma/irdma_umain.h index c67c5d7076f9..312310dcb754 100644 --- a/contrib/ofed/libirdma/irdma_umain.h +++ b/contrib/ofed/libirdma/irdma_umain.h @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef IRDMA_UMAIN_H #define IRDMA_UMAIN_H diff --git a/contrib/ofed/libirdma/irdma_uquery.h b/contrib/ofed/libirdma/irdma_uquery.h index cf56818e4d51..4660c05f0a91 100644 --- a/contrib/ofed/libirdma/irdma_uquery.h +++ b/contrib/ofed/libirdma/irdma_uquery.h @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef IRDMA_UQUERY_H diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h index 8eb9ae7990e7..0f0aa30bec91 100644 --- a/contrib/ofed/libirdma/irdma_user.h +++ b/contrib/ofed/libirdma/irdma_user.h @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef IRDMA_USER_H #define IRDMA_USER_H @@ -572,7 +571,6 @@ void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, u32 inline_data, u8 *shift); int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth); int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth); -int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth); void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta, u32 wqe_idx, bool post_sq); void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx); diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c index 14efab96a107..50fd2f23476d 100644 --- a/contrib/ofed/libirdma/irdma_uverbs.c +++ b/contrib/ofed/libirdma/irdma_uverbs.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (C) 2019 - 2022 Intel Corporation + * Copyright (C) 2019 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #include <config.h> #include <stdlib.h> @@ -266,11 +265,13 @@ irdma_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type) if (!mw) return NULL; - if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, - sizeof(resp))) { + err = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, + sizeof(resp)); + if (err) { printf("%s: Failed to alloc memory window\n", __func__); free(mw); + errno = err; return NULL; } @@ -446,8 +447,10 @@ ucreate_cq(struct ibv_context *context, if (!iwucq) return NULL; - if (pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE)) { + ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE); + if (ret) { free(iwucq); + errno = ret; return NULL; } @@ -464,8 +467,10 @@ ucreate_cq(struct ibv_context *context, iwucq->buf_size = total_size; info.cq_base = irdma_alloc_hw_buf(total_size); - if (!info.cq_base) + if (!info.cq_base) { + ret = ENOMEM; goto err_cq_base; + } memset(info.cq_base, 0, total_size); reg_mr_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ; @@ -476,17 +481,17 @@ ucreate_cq(struct ibv_context *context, IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr.ibv_mr, ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), ®_mr_resp, sizeof(reg_mr_resp)); - if (ret) { - errno = ret; + if (ret) goto err_dereg_mr; - } iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) { info.shadow_area = irdma_alloc_hw_buf(IRDMA_DB_SHADOW_AREA_SIZE); - if (!info.shadow_area) + if (!info.shadow_area) { + ret = ENOMEM; goto err_alloc_shadow; + } memset(info.shadow_area, 0, IRDMA_DB_SHADOW_AREA_SIZE); reg_mr_shadow_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ; @@ -499,7 +504,6 @@ ucreate_cq(struct ibv_context *context, ®_mr_shadow_resp, sizeof(reg_mr_shadow_resp)); if (ret) { irdma_free_hw_buf(info.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE); - errno = ret; goto err_alloc_shadow; } @@ -517,10 +521,8 @@ ucreate_cq(struct ibv_context *context, &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), &resp.ibv_resp, sizeof(resp.ibv_resp), sizeof(resp)); attr_ex->cqe = ncqe; - if (ret) { - errno = ret; + if (ret) goto err_create_cq; - } if (ext_cq) irdma_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex); @@ -548,6 +550,7 @@ err_cq_base: free(iwucq); + errno = ret; return NULL; } @@ -1560,7 +1563,8 @@ irdma_ucreate_qp(struct ibv_pd *pd, memset(iwuqp, 0, sizeof(*iwuqp)); - if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE)) + status = pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE); + if (status) goto err_free_qp; info.sq_size = info.sq_depth >> info.sq_shift; @@ -1575,35 +1579,37 @@ irdma_ucreate_qp(struct ibv_pd *pd, } iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges)); - if (!iwuqp->recv_sges) + if (!iwuqp->recv_sges) { + status = errno; /* preserve errno */ goto err_destroy_lock; + } info.wqe_alloc_db = (u32 *)iwvctx->db; info.legacy_mode = iwvctx->legacy_mode; info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array)); - if (!info.sq_wrtrk_array) + if (!info.sq_wrtrk_array) { + status = errno; /* preserve errno */ goto err_free_rsges; + } info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array)); - if (!info.rq_wrid_array) + if (!info.rq_wrid_array) { + status = errno; /* preserve errno */ goto err_free_sq_wrtrk; + } iwuqp->sq_sig_all = attr->sq_sig_all; iwuqp->qp_type = attr->qp_type; status = irdma_vmapped_qp(iwuqp, pd, attr, &info, iwvctx->legacy_mode); - if (status) { - errno = status; + if (status) goto err_free_rq_wrid; - } iwuqp->qp.back_qp = iwuqp; iwuqp->qp.lock = &iwuqp->lock; status = irdma_uk_qp_init(&iwuqp->qp, &info); - if (status) { - errno = status; + if (status) goto err_free_vmap_qp; - } attr->cap.max_send_wr = (info.sq_depth - IRDMA_SQ_RSVD) >> info.sq_shift; attr->cap.max_recv_wr = (info.rq_depth - IRDMA_RQ_RSVD) >> info.rq_shift; @@ -1625,6 +1631,7 @@ err_free_qp: printf("%s: failed to create QP\n", __func__); free(iwuqp); + errno = status; return NULL; } @@ -2081,11 +2088,10 @@ irdma_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr) struct irdma_ucreate_ah_resp resp = {}; int err; - err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, - &sgid); - if (err) { + if (ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, + &sgid)) { fprintf(stderr, "irdma: Error from ibv_query_gid.\n"); - errno = err; + errno = ENOENT; return NULL; } diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h index fc39c139b6ca..961e9cc99cf6 100644 --- a/contrib/ofed/libirdma/osdep.h +++ b/contrib/ofed/libirdma/osdep.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -31,7 +31,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -/*$FreeBSD$*/ #ifndef _ICRDMA_OSDEP_H_ #define _ICRDMA_OSDEP_H_ @@ -46,7 +45,6 @@ #include <sys/bus_dma.h> #include <sys/endian.h> -#define ATOMIC atomic_t #define IOMEM #define IRDMA_NTOHL(a) ntohl(a) #define IRDMA_NTOHS(a) ntohs(a) @@ -80,23 +78,13 @@ #define STATS_TIMER_DELAY 60000 /* a couple of linux size defines */ -#define SZ_128 128 -#define SZ_2K SZ_128 * 16 -#define SZ_1G (SZ_1K * SZ_1K * SZ_1K) -#define SPEED_1000 1000 -#define SPEED_10000 10000 -#define SPEED_20000 20000 -#define SPEED_25000 25000 -#define SPEED_40000 40000 -#define SPEED_100000 100000 - #define BIT_ULL(a) (1ULL << (a)) #define min(a, b) ((a) > (b) ? (b) : (a)) #ifndef likely -#define likely(x) __builtin_expect((x), 1) +#define likely(x) __builtin_expect((x), 1) #endif #ifndef unlikely -#define unlikely(x) __builtin_expect((x), 0) +#define unlikely(x) __builtin_expect((x), 0) #endif #define __aligned_u64 uint64_t __aligned(8) @@ -110,7 +98,7 @@ #define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__) #define irdma_debug_buf(dev, mask, desc, buf, size) \ do { \ - u32 i; \ + u32 i; \ if (!((mask) & (dev)->debug_mask)) { \ break; \ } \ @@ -120,20 +108,20 @@ do { \ irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \ } while(0) -#define irdma_debug(h, m, s, ...) \ -do { \ - if (!(h)) { \ - if ((m) == IRDMA_DEBUG_INIT) \ +#define irdma_debug(h, m, s, ...) \ +do { \ + if (!(h)) { \ + if ((m) == IRDMA_DEBUG_INIT) \ printf("irdma INIT " s, ##__VA_ARGS__); \ - } else if (((m) & (h)->debug_mask)) { \ - printf("irdma " s, ##__VA_ARGS__); \ - } \ + } else if (((m) & (h)->debug_mask)) { \ + printf("irdma " s, ##__VA_ARGS__); \ + } \ } while (0) extern unsigned int irdma_dbg; -#define libirdma_debug(fmt, args...) \ -do { \ - if (irdma_dbg) \ - printf("libirdma-%s: " fmt, __func__, ##args); \ +#define libirdma_debug(fmt, args...) \ +do { \ + if (irdma_dbg) \ + printf("libirdma-%s: " fmt, __func__, ##args); \ } while (0) #define irdma_dev_err(ibdev, fmt, ...) \ pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) @@ -142,18 +130,6 @@ do { \ #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf -#define dump_struct(s, sz, name) \ -do { \ - unsigned char *a; \ - printf("%s %u", (name), (unsigned int)(sz)); \ - for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \ - if ((u64)a % 8 == 0) \ - printf("\n%p ", a); \ - printf("%2x ", *a); \ - } \ - printf("\n"); \ -}while(0) - /* * debug definition end */ @@ -177,13 +153,11 @@ struct irdma_sc_vsi; #define rt_tos2priority(tos) (tos >> 5) #define ah_attr_to_dmac(attr) ((attr).dmac) -#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \ - ib_modify_qp_is_ok(cur_state, next_state, type, mask) #define kc_typeq_ib_wr const #define kc_ifp_find ip_ifp_find #define kc_ifp6_find ip6_ifp_find #define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \ - ib_gid_to_network_type(gid_type, gid) + ib_gid_to_network_type(gid_type, gid) #define irdma_del_timer_compat(tt) del_timer((tt)) #define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH #define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE @@ -191,36 +165,36 @@ struct irdma_sc_vsi; static inline void db_wr32(__u32 val, __u32 *wqe_word) { - *wqe_word = val; + *wqe_word = val; } void *hw_to_dev(struct irdma_hw *hw); struct irdma_dma_mem { - void *va; - u64 pa; + void *va; + u64 pa; bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_size_t size; - int nseg; - int flags; + int nseg; + int flags; }; struct irdma_virt_mem { - void *va; - u32 size; + void *va; + u32 size; }; #ifndef verbs_mr enum ibv_mr_type { - IBV_MR_TYPE_MR, - IBV_MR_TYPE_NULL_MR, + IBV_MR_TYPE_MR, + IBV_MR_TYPE_NULL_MR, }; struct verbs_mr { - struct ibv_mr ibv_mr; - enum ibv_mr_type mr_type; + struct ibv_mr ibv_mr; + enum ibv_mr_type mr_type; int access; }; #define verbs_get_mr(mr) container_of((mr), struct verbs_mr, ibv_mr) diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c index 4928b4189cca..5c11f7a35544 100644 --- a/sys/dev/irdma/fbsd_kcompat.c +++ b/sys/dev/irdma/fbsd_kcompat.c @@ -182,6 +182,31 @@ irdma_ieq_check_mpacrc(void *desc, return ret_code; } +static u_int +irdma_add_ipv6_cb(void *arg, struct ifaddr *addr, u_int count __unused){ + struct irdma_device *iwdev = arg; + struct sockaddr_in6 *sin6; + u32 local_ipaddr6[4] = {}; + char ip6buf[INET6_ADDRSTRLEN]; + u8 *mac_addr; + + sin6 = (struct sockaddr_in6 *)addr->ifa_addr; + + irdma_copy_ip_ntohl(local_ipaddr6, (u32 *)&sin6->sin6_addr); + + mac_addr = if_getlladdr(addr->ifa_ifp); + + printf("%s:%d IP=%s, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n", + __func__, __LINE__, + ip6_sprintf(ip6buf, &sin6->sin6_addr), + mac_addr[0], mac_addr[1], mac_addr[2], + mac_addr[3], mac_addr[4], mac_addr[5]); + + irdma_manage_arp_cache(iwdev->rf, mac_addr, local_ipaddr6, + IRDMA_ARP_ADD); + return (0); +} + /** * irdma_add_ipv6_addr - add ipv6 address to the hw arp table * @iwdev: irdma device @@ -190,32 +215,36 @@ irdma_ieq_check_mpacrc(void *desc, static void irdma_add_ipv6_addr(struct irdma_device *iwdev, struct ifnet *ifp) { - struct ifaddr *ifa, *tmp; - struct sockaddr_in6 *sin6; - u32 local_ipaddr6[4]; - u8 *mac_addr; - char ip6buf[INET6_ADDRSTRLEN]; - if_addr_rlock(ifp); - IRDMA_TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, tmp) { - sin6 = (struct sockaddr_in6 *)ifa->ifa_addr; - if (sin6->sin6_family != AF_INET6) - continue; + if_foreach_addr_type(ifp, AF_INET6, irdma_add_ipv6_cb, iwdev); + if_addr_runlock(ifp); +} - irdma_copy_ip_ntohl(local_ipaddr6, (u32 *)&sin6->sin6_addr); - mac_addr = IF_LLADDR(ifp); +static u_int +irdma_add_ipv4_cb(void *arg, struct ifaddr *addr, u_int count __unused){ + struct irdma_device *iwdev = arg; + struct sockaddr_in *sin; + u32 ip_addr[4] = {}; + uint8_t *mac_addr; - printf("%s:%d IP=%s, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n", - __func__, __LINE__, - ip6_sprintf(ip6buf, &sin6->sin6_addr), - mac_addr[0], mac_addr[1], mac_addr[2], - mac_addr[3], mac_addr[4], mac_addr[5]); + sin = (struct sockaddr_in *)addr->ifa_addr; - irdma_manage_arp_cache(iwdev->rf, mac_addr, local_ipaddr6, - IRDMA_ARP_ADD); + ip_addr[0] = ntohl(sin->sin_addr.s_addr); - } - if_addr_runlock(ifp); + mac_addr = if_getlladdr(addr->ifa_ifp); + + printf("%s:%d IP=%d.%d.%d.%d, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n", + __func__, __LINE__, + ip_addr[0] >> 24, + (ip_addr[0] >> 16) & 0xFF, + (ip_addr[0] >> 8) & 0xFF, + ip_addr[0] & 0xFF, + mac_addr[0], mac_addr[1], mac_addr[2], + mac_addr[3], mac_addr[4], mac_addr[5]); + + irdma_manage_arp_cache(iwdev->rf, mac_addr, ip_addr, + IRDMA_ARP_ADD); + return (0); } /** @@ -226,32 +255,8 @@ irdma_add_ipv6_addr(struct irdma_device *iwdev, struct ifnet *ifp) static void irdma_add_ipv4_addr(struct irdma_device *iwdev, struct ifnet *ifp) { - struct ifaddr *ifa; - struct sockaddr_in *sin; - u32 ip_addr[4] = {}; - u8 *mac_addr; - if_addr_rlock(ifp); - IRDMA_TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { - sin = (struct sockaddr_in *)ifa->ifa_addr; - if (sin->sin_family != AF_INET) - continue; - - ip_addr[0] = ntohl(sin->sin_addr.s_addr); - mac_addr = IF_LLADDR(ifp); - - printf("%s:%d IP=%d.%d.%d.%d, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n", - __func__, __LINE__, - ip_addr[0] >> 24, - (ip_addr[0] >> 16) & 0xFF, - (ip_addr[0] >> 8) & 0xFF, - ip_addr[0] & 0xFF, - mac_addr[0], mac_addr[1], mac_addr[2], - mac_addr[3], mac_addr[4], mac_addr[5]); - - irdma_manage_arp_cache(iwdev->rf, mac_addr, ip_addr, - IRDMA_ARP_ADD); - } + if_foreach_addr_type(ifp, AF_INET, irdma_add_ipv4_cb, iwdev); if_addr_runlock(ifp); } @@ -266,12 +271,15 @@ irdma_add_ip(struct irdma_device *iwdev) { struct ifnet *ifp = iwdev->netdev; struct ifnet *ifv; + struct epoch_tracker et; int i; irdma_add_ipv4_addr(iwdev, ifp); irdma_add_ipv6_addr(iwdev, ifp); - for (i = 0; ifp->if_vlantrunk != NULL && i < VLAN_N_VID; ++i) { + for (i = 0; if_getvlantrunk(ifp) != NULL && i < VLAN_N_VID; ++i) { + NET_EPOCH_ENTER(et); ifv = VLAN_DEVAT(ifp, i); + NET_EPOCH_EXIT(et); if (!ifv) continue; irdma_add_ipv4_addr(iwdev, ifv); @@ -292,7 +300,7 @@ irdma_ifaddrevent_handler(void *arg, struct ifnet *ifp, struct ifaddr *ifa, int if (!ifa || !ifa->ifa_addr || !ifp) return; if (rf->iwdev->netdev != ifp) { - for (i = 0; rf->iwdev->netdev->if_vlantrunk != NULL && i < VLAN_N_VID; ++i) { + for (i = 0; if_getvlantrunk(rf->iwdev->netdev) != NULL && i < VLAN_N_VID; ++i) { NET_EPOCH_ENTER(et); ifv = VLAN_DEVAT(rf->iwdev->netdev, i); NET_EPOCH_EXIT(et); @@ -354,7 +362,8 @@ irdma_get_route_ifp(struct sockaddr *dst_sin, struct ifnet *netdev, struct nhop_object *nh; if (dst_sin->sa_family == AF_INET6) - nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, 0, NHR_NONE, 0); + nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, + ((struct sockaddr_in6 *)dst_sin)->sin6_scope_id, NHR_NONE, 0); else nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0); if (!nh || (nh->nh_ifp != netdev && @@ -466,7 +475,7 @@ irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node *cm_node, dst_addr.sin6_family = AF_INET6; dst_addr.sin6_len = sizeof(dst_addr); - dst_addr.sin6_scope_id = iwdev->netdev->if_index; + dst_addr.sin6_scope_id = if_getindex(iwdev->netdev); irdma_copy_ip_htonl(dst_addr.sin6_addr.__u6_addr.__u6_addr32, dest); err = irdma_get_dst_mac(cm_node, (struct sockaddr *)&dst_addr, dst_mac); *** 5195 LINES SKIPPED ***
