Gitweb:     
http://git.kernel.org/git/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=6cff2faaf135b602c914710f3414630c3fc1ee83
Commit:     6cff2faaf135b602c914710f3414630c3fc1ee83
Parent:     d42b01b584b6f55f70c56f6a3dabc26f4982d30d
Author:     Ralph Campbell <[EMAIL PROTECTED]>
AuthorDate: Fri Sep 7 16:54:01 2007 -0700
Committer:  Roland Dreier <[EMAIL PROTECTED]>
CommitDate: Tue Oct 9 20:52:23 2007 -0700

    IB/ipath: Optimize completion queue entry insertion and polling
    
    The code to add an entry to the completion queue stored the QPN which is
    needed for the user level verbs view of the completion queue entry but
    the kernel struct ib_wc contains a pointer to the QP instead of a QPN.
    When the kernel polled for a completion queue entry, the QPN was lookup
    up and the QP pointer recovered. This patch stores the CQE differently
    based on whether the CQ is a kernel CQ or a user CQ thus avoiding the
    QPN to QP lookup overhead.
    
    Signed-off-by: Ralph Campbell <[EMAIL PROTECTED]>
    Signed-off-by: Roland Dreier <[EMAIL PROTECTED]>
---
 drivers/infiniband/hw/ipath/ipath_cq.c    |   94 +++++++++++++++--------------
 drivers/infiniband/hw/ipath/ipath_verbs.h |    6 ++-
 2 files changed, 53 insertions(+), 47 deletions(-)

diff --git a/drivers/infiniband/hw/ipath/ipath_cq.c 
b/drivers/infiniband/hw/ipath/ipath_cq.c
index a6f04d2..645ed71 100644
--- a/drivers/infiniband/hw/ipath/ipath_cq.c
+++ b/drivers/infiniband/hw/ipath/ipath_cq.c
@@ -76,22 +76,25 @@ void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc 
*entry, int solicited)
                }
                return;
        }
-       wc->queue[head].wr_id = entry->wr_id;
-       wc->queue[head].status = entry->status;
-       wc->queue[head].opcode = entry->opcode;
-       wc->queue[head].vendor_err = entry->vendor_err;
-       wc->queue[head].byte_len = entry->byte_len;
-       wc->queue[head].imm_data = (__u32 __force)entry->imm_data;
-       wc->queue[head].qp_num = entry->qp->qp_num;
-       wc->queue[head].src_qp = entry->src_qp;
-       wc->queue[head].wc_flags = entry->wc_flags;
-       wc->queue[head].pkey_index = entry->pkey_index;
-       wc->queue[head].slid = entry->slid;
-       wc->queue[head].sl = entry->sl;
-       wc->queue[head].dlid_path_bits = entry->dlid_path_bits;
-       wc->queue[head].port_num = entry->port_num;
-       /* Make sure queue entry is written before the head index. */
-       smp_wmb();
+       if (cq->ip) {
+               wc->uqueue[head].wr_id = entry->wr_id;
+               wc->uqueue[head].status = entry->status;
+               wc->uqueue[head].opcode = entry->opcode;
+               wc->uqueue[head].vendor_err = entry->vendor_err;
+               wc->uqueue[head].byte_len = entry->byte_len;
+               wc->uqueue[head].imm_data = (__u32 __force)entry->imm_data;
+               wc->uqueue[head].qp_num = entry->qp->qp_num;
+               wc->uqueue[head].src_qp = entry->src_qp;
+               wc->uqueue[head].wc_flags = entry->wc_flags;
+               wc->uqueue[head].pkey_index = entry->pkey_index;
+               wc->uqueue[head].slid = entry->slid;
+               wc->uqueue[head].sl = entry->sl;
+               wc->uqueue[head].dlid_path_bits = entry->dlid_path_bits;
+               wc->uqueue[head].port_num = entry->port_num;
+               /* Make sure entry is written before the head index. */
+               smp_wmb();
+       } else
+               wc->kqueue[head] = *entry;
        wc->head = next;
 
        if (cq->notify == IB_CQ_NEXT_COMP ||
@@ -130,6 +133,12 @@ int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, 
struct ib_wc *entry)
        int npolled;
        u32 tail;
 
+       /* The kernel can only poll a kernel completion queue */
+       if (cq->ip) {
+               npolled = -EINVAL;
+               goto bail;
+       }
+
        spin_lock_irqsave(&cq->lock, flags);
 
        wc = cq->queue;
@@ -137,31 +146,10 @@ int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, 
struct ib_wc *entry)
        if (tail > (u32) cq->ibcq.cqe)
                tail = (u32) cq->ibcq.cqe;
        for (npolled = 0; npolled < num_entries; ++npolled, ++entry) {
-               struct ipath_qp *qp;
-
                if (tail == wc->head)
                        break;
-               /* Make sure entry is read after head index is read. */
-               smp_rmb();
-               qp = ipath_lookup_qpn(&to_idev(cq->ibcq.device)->qp_table,
-                                     wc->queue[tail].qp_num);
-               entry->qp = &qp->ibqp;
-               if (atomic_dec_and_test(&qp->refcount))
-                       wake_up(&qp->wait);
-
-               entry->wr_id = wc->queue[tail].wr_id;
-               entry->status = wc->queue[tail].status;
-               entry->opcode = wc->queue[tail].opcode;
-               entry->vendor_err = wc->queue[tail].vendor_err;
-               entry->byte_len = wc->queue[tail].byte_len;
-               entry->imm_data = wc->queue[tail].imm_data;
-               entry->src_qp = wc->queue[tail].src_qp;
-               entry->wc_flags = wc->queue[tail].wc_flags;
-               entry->pkey_index = wc->queue[tail].pkey_index;
-               entry->slid = wc->queue[tail].slid;
-               entry->sl = wc->queue[tail].sl;
-               entry->dlid_path_bits = wc->queue[tail].dlid_path_bits;
-               entry->port_num = wc->queue[tail].port_num;
+               /* The kernel doesn't need a RMB since it has the lock. */
+               *entry = wc->kqueue[tail];
                if (tail >= cq->ibcq.cqe)
                        tail = 0;
                else
@@ -171,6 +159,7 @@ int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, 
struct ib_wc *entry)
 
        spin_unlock_irqrestore(&cq->lock, flags);
 
+bail:
        return npolled;
 }
 
@@ -215,6 +204,7 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int 
entries, int comp_vec
        struct ipath_cq *cq;
        struct ipath_cq_wc *wc;
        struct ib_cq *ret;
+       u32 sz;
 
        if (entries < 1 || entries > ib_ipath_max_cqes) {
                ret = ERR_PTR(-EINVAL);
@@ -235,7 +225,12 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int 
entries, int comp_vec
         * We need to use vmalloc() in order to support mmap and large
         * numbers of entries.
         */
-       wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * entries);
+       sz = sizeof(*wc);
+       if (udata && udata->outlen >= sizeof(__u64))
+               sz += sizeof(struct ib_uverbs_wc) * (entries + 1);
+       else
+               sz += sizeof(struct ib_wc) * (entries + 1);
+       wc = vmalloc_user(sz);
        if (!wc) {
                ret = ERR_PTR(-ENOMEM);
                goto bail_cq;
@@ -247,9 +242,8 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int 
entries, int comp_vec
         */
        if (udata && udata->outlen >= sizeof(__u64)) {
                int err;
-               u32 s = sizeof *wc + sizeof(struct ib_wc) * entries;
 
-               cq->ip = ipath_create_mmap_info(dev, s, context, wc);
+               cq->ip = ipath_create_mmap_info(dev, sz, context, wc);
                if (!cq->ip) {
                        ret = ERR_PTR(-ENOMEM);
                        goto bail_wc;
@@ -380,6 +374,7 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct 
ib_udata *udata)
        struct ipath_cq_wc *wc;
        u32 head, tail, n;
        int ret;
+       u32 sz;
 
        if (cqe < 1 || cqe > ib_ipath_max_cqes) {
                ret = -EINVAL;
@@ -389,7 +384,12 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct 
ib_udata *udata)
        /*
         * Need to use vmalloc() if we want to support large #s of entries.
         */
-       wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * cqe);
+       sz = sizeof(*wc);
+       if (udata && udata->outlen >= sizeof(__u64))
+               sz += sizeof(struct ib_uverbs_wc) * (cqe + 1);
+       else
+               sz += sizeof(struct ib_wc) * (cqe + 1);
+       wc = vmalloc_user(sz);
        if (!wc) {
                ret = -ENOMEM;
                goto bail;
@@ -430,7 +430,10 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct 
ib_udata *udata)
                goto bail;
        }
        for (n = 0; tail != head; n++) {
-               wc->queue[n] = old_wc->queue[tail];
+               if (cq->ip)
+                       wc->uqueue[n] = old_wc->uqueue[tail];
+               else
+                       wc->kqueue[n] = old_wc->kqueue[tail];
                if (tail == (u32) cq->ibcq.cqe)
                        tail = 0;
                else
@@ -447,9 +450,8 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct 
ib_udata *udata)
        if (cq->ip) {
                struct ipath_ibdev *dev = to_idev(ibcq->device);
                struct ipath_mmap_info *ip = cq->ip;
-               u32 s = sizeof *wc + sizeof(struct ib_wc) * cqe;
 
-               ipath_update_mmap_info(dev, ip, s, wc);
+               ipath_update_mmap_info(dev, ip, sz, wc);
                spin_lock_irq(&dev->pending_lock);
                if (list_empty(&ip->pending_mmaps))
                        list_add(&ip->pending_mmaps, &dev->pending_mmaps);
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.h 
b/drivers/infiniband/hw/ipath/ipath_verbs.h
index a197229..9be9bf9 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.h
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.h
@@ -191,7 +191,11 @@ struct ipath_mmap_info {
 struct ipath_cq_wc {
        u32 head;               /* index of next entry to fill */
        u32 tail;               /* index of next ib_poll_cq() entry */
-       struct ib_uverbs_wc queue[1]; /* this is actually size ibcq.cqe + 1 */
+       union {
+               /* these are actually size ibcq.cqe + 1 */
+               struct ib_uverbs_wc uqueue[0];
+               struct ib_wc kqueue[0];
+       };
 };
 
 /*
-
To unsubscribe from this list: send the line "unsubscribe git-commits-head" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Reply via email to