[PATCH 3.2 055/107] IB/qib: Change lkey table allocation to support more MRs

[Ben Hutchings]

3.2.72-rc1 review patch.  If anyone has any objections, please let me know.

--

From: Mike Marciniszyn 

commit d6f1c17e162b2a11e708f28fa93f2f79c164b442 upstream.

The lkey table is allocated with with a get_user_pages() with an
order based on a number of index bits from a module parameter.

The underlying kernel code cannot allocate that many contiguous pages.

There is no reason the underlying memory needs to be physically
contiguous.

This patch:
- switches the allocation/deallocation to vmalloc/vfree
- caps the number of bits to 23 to insure at least 1 generation bit
  o this matches the module parameter description

Reviewed-by: Vinit Agnihotri 
Signed-off-by: Mike Marciniszyn 
Signed-off-by: Doug Ledford 
[bwh: Backported to 3.2:
 - Adjust context
 - Add definition of qib_dev_warn(), added upstream by commit ddb887658970
   ("IB/qib: Convert opcode counters to per-context")]
Signed-off-by: Ben Hutchings 
---
 drivers/infiniband/hw/qib/qib_keys.c  |  4 
 drivers/infiniband/hw/qib/qib_verbs.c | 14 ++
 drivers/infiniband/hw/qib/qib_verbs.h |  2 ++
 3 files changed, 16 insertions(+), 4 deletions(-)

--- a/drivers/infiniband/hw/qib/qib_keys.c
+++ b/drivers/infiniband/hw/qib/qib_keys.c
@@ -69,6 +69,10 @@ int qib_alloc_lkey(struct qib_lkey_table
 * unrestricted LKEY.
 */
rkt->gen++;
+   /*
+* bits are capped in qib_verbs.c to insure enough bits
+* for generation number
+*/
mr->lkey = (r << (32 - ib_qib_lkey_table_size)) |
1 << (24 - ib_qib_lkey_table_size)) - 1) & rkt->gen)
 << 8);
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -40,6 +40,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "qib.h"
 #include "qib_common.h"
@@ -2035,10 +2036,16 @@ int qib_register_ib_device(struct qib_de
 * the LKEY).  The remaining bits act as a generation number or tag.
 */
spin_lock_init(>lk_table.lock);
+   /* insure generation is at least 4 bits see keys.c */
+   if (ib_qib_lkey_table_size > MAX_LKEY_TABLE_BITS) {
+   qib_dev_warn(dd, "lkey bits %u too large, reduced to %u\n",
+   ib_qib_lkey_table_size, MAX_LKEY_TABLE_BITS);
+   ib_qib_lkey_table_size = MAX_LKEY_TABLE_BITS;
+   }
dev->lk_table.max = 1 << ib_qib_lkey_table_size;
lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
dev->lk_table.table = (struct qib_mregion **)
-   __get_free_pages(GFP_KERNEL, get_order(lk_tab_size));
+   vmalloc(lk_tab_size);
if (dev->lk_table.table == NULL) {
ret = -ENOMEM;
goto err_lk;
@@ -2208,7 +2215,7 @@ err_tx:
sizeof(struct qib_pio_header),
  dev->pio_hdrs, dev->pio_hdrs_phys);
 err_hdrs:
-   free_pages((unsigned long) dev->lk_table.table, get_order(lk_tab_size));
+   vfree(dev->lk_table.table);
 err_lk:
kfree(dev->qp_table);
 err_qpt:
@@ -2262,7 +2269,6 @@ void qib_unregister_ib_device(struct qib
sizeof(struct qib_pio_header),
  dev->pio_hdrs, dev->pio_hdrs_phys);
lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
-   free_pages((unsigned long) dev->lk_table.table,
-  get_order(lk_tab_size));
+   vfree(dev->lk_table.table);
kfree(dev->qp_table);
 }
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -622,6 +622,8 @@ struct qib_qpn_table {
struct qpn_map map[QPNMAP_ENTRIES];
 };
 
+#define MAX_LKEY_TABLE_BITS 23
+
 struct qib_lkey_table {
spinlock_t lock; /* protect changes in this struct */
u32 next;   /* next unused index (speeds search) */
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -1421,6 +1421,10 @@ extern struct mutex qib_mutex;
qib_get_unit_name((dd)->unit), ##__VA_ARGS__); \
} while (0)
 
+#define qib_dev_warn(dd, fmt, ...) \
+   dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
+   qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
+
 #define qib_dev_porterr(dd, port, fmt, ...) \
do { \
dev_err(&(dd)->pcidev->dev, "%s: IB%u:%u " fmt, \

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

[PATCH 3.2 055/107] IB/qib: Change lkey table allocation to support more MRs

[Ben Hutchings]

3.2.72-rc1 review patch.  If anyone has any objections, please let me know.

--

From: Mike Marciniszyn 

commit d6f1c17e162b2a11e708f28fa93f2f79c164b442 upstream.

The lkey table is allocated with with a get_user_pages() with an
order based on a number of index bits from a module parameter.

The underlying kernel code cannot allocate that many contiguous pages.

There is no reason the underlying memory needs to be physically
contiguous.

This patch:
- switches the allocation/deallocation to vmalloc/vfree
- caps the number of bits to 23 to insure at least 1 generation bit
  o this matches the module parameter description

Reviewed-by: Vinit Agnihotri 
Signed-off-by: Mike Marciniszyn 
Signed-off-by: Doug Ledford 
[bwh: Backported to 3.2:
 - Adjust context
 - Add definition of qib_dev_warn(), added upstream by commit ddb887658970
   ("IB/qib: Convert opcode counters to per-context")]
Signed-off-by: Ben Hutchings 
---
 drivers/infiniband/hw/qib/qib_keys.c  |  4 
 drivers/infiniband/hw/qib/qib_verbs.c | 14 ++
 drivers/infiniband/hw/qib/qib_verbs.h |  2 ++
 3 files changed, 16 insertions(+), 4 deletions(-)

--- a/drivers/infiniband/hw/qib/qib_keys.c
+++ b/drivers/infiniband/hw/qib/qib_keys.c
@@ -69,6 +69,10 @@ int qib_alloc_lkey(struct qib_lkey_table
 * unrestricted LKEY.
 */
rkt->gen++;
+   /*
+* bits are capped in qib_verbs.c to insure enough bits
+* for generation number
+*/
mr->lkey = (r << (32 - ib_qib_lkey_table_size)) |
1 << (24 - ib_qib_lkey_table_size)) - 1) & rkt->gen)
 << 8);
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -40,6 +40,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "qib.h"
 #include "qib_common.h"
@@ -2035,10 +2036,16 @@ int qib_register_ib_device(struct qib_de
 * the LKEY).  The remaining bits act as a generation number or tag.
 */
spin_lock_init(>lk_table.lock);
+   /* insure generation is at least 4 bits see keys.c */
+   if (ib_qib_lkey_table_size > MAX_LKEY_TABLE_BITS) {
+   qib_dev_warn(dd, "lkey bits %u too large, reduced to %u\n",
+   ib_qib_lkey_table_size, MAX_LKEY_TABLE_BITS);
+   ib_qib_lkey_table_size = MAX_LKEY_TABLE_BITS;
+   }
dev->lk_table.max = 1 << ib_qib_lkey_table_size;
lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
dev->lk_table.table = (struct qib_mregion **)
-   __get_free_pages(GFP_KERNEL, get_order(lk_tab_size));
+   vmalloc(lk_tab_size);
if (dev->lk_table.table == NULL) {
ret = -ENOMEM;
goto err_lk;
@@ -2208,7 +2215,7 @@ err_tx:
sizeof(struct qib_pio_header),
  dev->pio_hdrs, dev->pio_hdrs_phys);
 err_hdrs:
-   free_pages((unsigned long) dev->lk_table.table, get_order(lk_tab_size));
+   vfree(dev->lk_table.table);
 err_lk:
kfree(dev->qp_table);
 err_qpt:
@@ -2262,7 +2269,6 @@ void qib_unregister_ib_device(struct qib
sizeof(struct qib_pio_header),
  dev->pio_hdrs, dev->pio_hdrs_phys);
lk_tab_size = dev->lk_table.max * sizeof(*dev->lk_table.table);
-   free_pages((unsigned long) dev->lk_table.table,
-  get_order(lk_tab_size));
+   vfree(dev->lk_table.table);
kfree(dev->qp_table);
 }
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -622,6 +622,8 @@ struct qib_qpn_table {
struct qpn_map map[QPNMAP_ENTRIES];
 };
 
+#define MAX_LKEY_TABLE_BITS 23
+
 struct qib_lkey_table {
spinlock_t lock; /* protect changes in this struct */
u32 next;   /* next unused index (speeds search) */
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -1421,6 +1421,10 @@ extern struct mutex qib_mutex;
qib_get_unit_name((dd)->unit), ##__VA_ARGS__); \
} while (0)
 
+#define qib_dev_warn(dd, fmt, ...) \
+   dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
+   qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
+
 #define qib_dev_porterr(dd, port, fmt, ...) \
do { \
dev_err(&(dd)->pcidev->dev, "%s: IB%u:%u " fmt, \

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/