Quoting r. Roland Dreier <[EMAIL PROTECTED]>:
> Subject: strange mem-free bug (was: [openib-general] completion Q overflow
> error/panic)
>
> While looking at Viswa's example, I've found what seems to be a
> problem using lots of QPs on mem-free HCAs.
Hi, Roland!
This seems to be a bug in libmthca. Patch below.
We probably need a similiar fix for kernel mthca - let me know if
you plan to work on that, otherwise I'll look into it tomorrow.
And its probably something we want fixed for 2.6.14, right?
Let me know.
With regard to the test code that you posted - I also have some small
comments. If you plan to use it in the future, you can stick it
in svn somewhere and I'll send patches.
---
Fix posting of the first work request for memfree hardware.
Simplify code for tavor mode hardware.
Signed-off-by: Michael S. Tsirkin <[EMAIL PROTECTED]>
Index: userspace/libmthca/src/qp.c
===================================================================
--- userspace.orig/libmthca/src/qp.c 2005-09-13 17:17:58.000000000 +0300
+++ userspace/libmthca/src/qp.c 2005-09-13 17:26:23.000000000 +0300
@@ -259,15 +259,13 @@ int mthca_tavor_post_send(struct ibv_qp
goto out;
}
- if (prev_wqe) {
- ((struct mthca_next_seg *) prev_wqe)->nda_op =
- htonl(((ind << qp->sq.wqe_shift) +
- qp->send_wqe_offset) |
- mthca_opcode[wr->opcode]);
+ ((struct mthca_next_seg *) prev_wqe)->nda_op =
+ htonl(((ind << qp->sq.wqe_shift) +
+ qp->send_wqe_offset) |
+ mthca_opcode[wr->opcode]);
- ((struct mthca_next_seg *) prev_wqe)->ee_nds =
- htonl((size0 ? 0 : MTHCA_NEXT_DBD) | size);
- }
+ ((struct mthca_next_seg *) prev_wqe)->ee_nds =
+ htonl((size0 ? 0 : MTHCA_NEXT_DBD) | size);
if (!size0) {
size0 = size;
@@ -353,12 +351,10 @@ int mthca_tavor_post_recv(struct ibv_qp
qp->wrid[ind] = wr->wr_id;
- if (prev_wqe) {
- ((struct mthca_next_seg *) prev_wqe)->nda_op =
- htonl((ind << qp->rq.wqe_shift) | 1);
- ((struct mthca_next_seg *) prev_wqe)->ee_nds =
- htonl(MTHCA_NEXT_DBD | size);
- }
+ ((struct mthca_next_seg *) prev_wqe)->nda_op =
+ htonl((ind << qp->rq.wqe_shift) | 1);
+ ((struct mthca_next_seg *) prev_wqe)->ee_nds =
+ htonl(MTHCA_NEXT_DBD | size);
if (!size0)
size0 = size;
@@ -562,15 +558,13 @@ int mthca_arbel_post_send(struct ibv_qp
goto out;
}
- if (prev_wqe) {
- ((struct mthca_next_seg *) prev_wqe)->nda_op =
- htonl(((ind << qp->sq.wqe_shift) +
- qp->send_wqe_offset) |
- mthca_opcode[wr->opcode]);
- mb();
- ((struct mthca_next_seg *) prev_wqe)->ee_nds =
- htonl(MTHCA_NEXT_DBD | size);
- }
+ ((struct mthca_next_seg *) prev_wqe)->nda_op =
+ htonl(((ind << qp->sq.wqe_shift) +
+ qp->send_wqe_offset) |
+ mthca_opcode[wr->opcode]);
+ mb();
+ ((struct mthca_next_seg *) prev_wqe)->ee_nds =
+ htonl(MTHCA_NEXT_DBD | size);
if (!size0) {
size0 = size;
@@ -767,6 +761,8 @@ int mthca_alloc_qp_buf(struct ibv_pd *pd
}
}
+ qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);
+ qp->rq.last = get_recv_wqe(qp, qp->sq.max - 1);
return 0;
}
Index: userspace/libmthca/src/srq.c
===================================================================
--- userspace.orig/libmthca/src/srq.c 2005-09-13 17:25:41.000000000 +0300
+++ userspace/libmthca/src/srq.c 2005-09-13 17:25:51.000000000 +0300
@@ -142,13 +142,11 @@ int mthca_tavor_post_srq_recv(struct ibv
((struct mthca_data_seg *) wqe)->addr = 0;
}
- if (prev_wqe) {
- ((struct mthca_next_seg *) prev_wqe)->nda_op =
- htonl((ind << srq->wqe_shift) | 1);
- mb();
- ((struct mthca_next_seg *) prev_wqe)->ee_nds =
- htonl(MTHCA_NEXT_DBD);
- }
+ ((struct mthca_next_seg *) prev_wqe)->nda_op =
+ htonl((ind << srq->wqe_shift) | 1);
+ mb();
+ ((struct mthca_next_seg *) prev_wqe)->ee_nds =
+ htonl(MTHCA_NEXT_DBD);
srq->wrid[ind] = wr->wr_id;
srq->first_free = next_ind;
@@ -294,6 +292,7 @@ int mthca_alloc_srq_buf(struct ibv_pd *p
srq->first_free = 0;
srq->last_free = srq->max - 1;
+ srq->last = get_wqe(srq, srq->max - 1);
return 0;
}
Index: userspace/libmthca/src/verbs.c
===================================================================
--- userspace.orig/libmthca/src/verbs.c 2005-08-23 14:03:12.000000000 +0300
+++ userspace/libmthca/src/verbs.c 2005-09-13 17:25:14.000000000 +0300
@@ -306,7 +306,6 @@ struct ibv_srq *mthca_create_srq(struct
srq->max = align_queue_size(pd->context, attr->attr.max_wr, 1);
srq->max_gs = attr->attr.max_sge;
- srq->last = NULL;
srq->counter = 0;
if (mthca_alloc_srq_buf(pd, &attr->attr, srq))
@@ -413,14 +412,12 @@ struct ibv_qp *mthca_create_qp(struct ib
qp->sq.last_comp = qp->sq.max - 1;
qp->sq.head = 0;
qp->sq.tail = 0;
- qp->sq.last = NULL;
qp->rq.max = align_queue_size(pd->context, attr->cap.max_recv_wr,
0);
qp->rq.next_ind = 0;
qp->rq.last_comp = qp->rq.max - 1;
qp->rq.head = 0;
qp->rq.tail = 0;
- qp->rq.last = NULL;
if (mthca_alloc_qp_buf(pd, &attr->cap, qp))
goto err;
--
MST
_______________________________________________
openib-general mailing list
[email protected]
http://openib.org/mailman/listinfo/openib-general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
