The current driver checks required inline size by making sure it does not exceed 1024. This is wrong since the whole WQE is limited to 1008 bytes. Moreover, a more careful claculation is required to avoid cases where the application requests inline support in a certain size that when used later could cause connections to stall due to bad WQEs. This patch takes into account the size of the WQE, the segements used to create a WQE and the overhead incured by the inline segments themselves.
Signed-off-by: Eli Cohen <[email protected]> --- Changes from previous version: Use the existing num_inline_segs() function in clacualting the max possible inline size in verify_sizes() Use a macro to define the max WQE size. There is a complementary patch following this one that fixes yet another problem related to inline cacluation. src/mlx4.h | 5 +++++ src/qp.c | 2 +- src/verbs.c | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/mlx4.h b/src/mlx4.h index 4445998..5223697 100644 --- a/src/mlx4.h +++ b/src/mlx4.h @@ -130,6 +130,10 @@ enum { MLX4_CQE_OPCODE_RESIZE = 0x16, }; +enum { + MLX4_MAX_WQE_SIZE = 1008 +}; + struct mlx4_device { struct ibv_device ibv_dev; int page_size; @@ -349,6 +353,7 @@ int mlx4_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr); void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type, struct mlx4_qp *qp); +int num_inline_segs(int data, enum ibv_qp_type type); int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap, enum ibv_qp_type type, struct mlx4_qp *qp); void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap, diff --git a/src/qp.c b/src/qp.c index d194ae3..6ceb3ef 100644 --- a/src/qp.c +++ b/src/qp.c @@ -495,7 +495,7 @@ out: return ret; } -static int num_inline_segs(int data, enum ibv_qp_type type) +int num_inline_segs(int data, enum ibv_qp_type type) { /* * Inline data segments are not allowed to cross 64 byte diff --git a/src/verbs.c b/src/verbs.c index 1ac1362..5ab20e5 100644 --- a/src/verbs.c +++ b/src/verbs.c @@ -384,6 +384,49 @@ int mlx4_destroy_srq(struct ibv_srq *srq) return 0; } +static int verify_sizes(struct ibv_qp_init_attr *attr) +{ + int size; + int nsegs; + + if (attr->cap.max_send_wr > 65536 || + attr->cap.max_recv_wr > 65536 || + attr->cap.max_send_sge > 64 || + attr->cap.max_recv_sge > 64) + return -1; + + /* + * basic numbers needed to understand the calculation + * 1008 is max size of a WQE. + * 64 is a cache line + * 4 bytes for inline header + */ + if (attr->cap.max_inline_data) { + nsegs = num_inline_segs(attr->cap.max_inline_data, attr->qp_type); + size = MLX4_MAX_WQE_SIZE - nsegs * sizeof(struct mlx4_wqe_inline_seg); + switch (attr->qp_type) { + case IBV_QPT_UD: + size -= (sizeof(struct mlx4_wqe_ctrl_seg) + + sizeof(struct mlx4_wqe_datagram_seg)); + break; + + case IBV_QPT_RC: + case IBV_QPT_UC: + size -= (sizeof(struct mlx4_wqe_ctrl_seg) + + sizeof(struct mlx4_wqe_raddr_seg)); + break; + + default: + return 0; + } + + if (attr->cap.max_inline_data > size) + return -1; + } + + return 0; +} + struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) { struct mlx4_create_qp cmd; @@ -392,11 +435,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) int ret; /* Sanity check QP size before proceeding */ - if (attr->cap.max_send_wr > 65536 || - attr->cap.max_recv_wr > 65536 || - attr->cap.max_send_sge > 64 || - attr->cap.max_recv_sge > 64 || - attr->cap.max_inline_data > 1024) + if (verify_sizes(attr)) return NULL; qp = malloc(sizeof *qp); -- 1.7.3 -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
