Thanks, I committed the following patch. I fixed the computation
+ qp->sq.max_gs = align(cap->max_inline_data +
+ sizeof (struct mthca_inline_seg),
+ sizeof (struct mthca_data_seg));
to divide by sizeof (struct mthca_data_seg) -- otherwise it seems that
max_gs will end up far too large. I also added some code to return
the actual values allocated back to the consumer.
- R.
--- libmthca/src/qp.c (revision 2206)
+++ libmthca/src/qp.c (working copy)
@@ -718,10 +718,17 @@ out:
return ret;
}
-int mthca_alloc_qp_buf(struct ibv_pd *pd, struct mthca_qp *qp)
+int mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
+ struct mthca_qp *qp)
{
int size;
+ qp->rq.max_gs = cap->max_recv_sge;
+ qp->sq.max_gs = align(cap->max_inline_data + sizeof (struct
mthca_inline_seg),
+ sizeof (struct mthca_data_seg)) / sizeof
(struct mthca_data_seg);
+ if (qp->sq.max_gs < cap->max_send_sge)
+ qp->sq.max_gs = cap->max_send_sge;
+
qp->wrid = malloc((qp->rq.max + qp->sq.max) * sizeof (uint64_t));
if (!qp->wrid)
return -1;
@@ -796,6 +803,39 @@ int mthca_alloc_qp_buf(struct ibv_pd *pd
return 0;
}
+void mthca_return_cap(struct ibv_pd *pd, struct mthca_qp *qp, struct
ibv_qp_cap *cap)
+{
+ /*
+ * Maximum inline data size is the full WQE size less the size
+ * of the next segment, inline segment and other non-data segments.
+ */
+ cap->max_inline_data = (1 << qp->sq.wqe_shift) -
+ sizeof (struct mthca_next_seg) -
+ sizeof (struct mthca_inline_seg);
+
+ switch (qp->qpt) {
+ case IBV_QPT_UD:
+ if (mthca_is_memfree(pd->context))
+ cap->max_inline_data -= sizeof (struct
mthca_arbel_ud_seg);
+ else
+ cap->max_inline_data -= sizeof (struct
mthca_tavor_ud_seg);
+ break;
+
+ default:
+ /*
+ * inline data won't be used in the same WQE as an
+ * atomic or bind segment, so we don't have to
+ * subtract anything off here.
+ */
+ break;
+ }
+
+ cap->max_send_wr = qp->sq.max;
+ cap->max_recv_wr = qp->rq.max;
+ cap->max_send_sge = qp->sq.max_gs;
+ cap->max_recv_sge = qp->rq.max_gs;
+}
+
struct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn)
{
int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;
--- libmthca/src/verbs.c (revision 2206)
+++ libmthca/src/verbs.c (working copy)
@@ -270,7 +270,6 @@ struct ibv_qp *mthca_create_qp(struct ib
qp->qpt = attr->qp_type;
qp->sq.max = align_qp_size(pd->context, attr->cap.max_send_wr);
- qp->sq.max_gs = attr->cap.max_send_sge;
qp->sq.next_ind = 0;
qp->sq.last_comp = qp->sq.max - 1;
qp->sq.head = 0;
@@ -278,14 +277,13 @@ struct ibv_qp *mthca_create_qp(struct ib
qp->sq.last = NULL;
qp->rq.max = align_qp_size(pd->context, attr->cap.max_recv_wr);
- qp->rq.max_gs = attr->cap.max_recv_sge;
qp->rq.next_ind = 0;
qp->rq.last_comp = qp->rq.max - 1;
qp->rq.head = 0;
qp->rq.tail = 0;
qp->rq.last = NULL;
- if (mthca_alloc_qp_buf(pd, qp))
+ if (mthca_alloc_qp_buf(pd, &attr->cap, qp))
goto err;
if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) ||
@@ -332,6 +330,8 @@ struct ibv_qp *mthca_create_qp(struct ib
if (ret)
goto err_destroy;
+ mthca_return_cap(pd, qp, &attr->cap);
+
return &qp->ibv_qp;
err_destroy:
--- libmthca/src/mthca.h (revision 2190)
+++ libmthca/src/mthca.h (working copy)
@@ -284,7 +284,10 @@ extern int mthca_arbel_post_send(struct
struct ibv_send_wr **bad_wr);
extern int mthca_arbel_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
struct ibv_recv_wr **bad_wr);
-extern int mthca_alloc_qp_buf(struct ibv_pd *pd, struct mthca_qp *qp);
+extern int mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
+ struct mthca_qp *qp);
+extern void mthca_return_cap(struct ibv_pd *pd, struct mthca_qp *qp,
+ struct ibv_qp_cap *cap);
extern struct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn);
extern int mthca_store_qp(struct mthca_context *ctx, uint32_t qpn, struct
mthca_qp *qp);
extern void mthca_clear_qp(struct mthca_context *ctx, uint32_t qpn);
_______________________________________________
openib-general mailing list
[email protected]
http://openib.org/mailman/listinfo/openib-general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
