Re: [PATCH 26/37] IB/rdmavt: Move memory registration into rdmavt

[Sagi Grimberg]

  /**
@@ -83,7 +357,31 @@ struct ib_mr *rvt_reg_phys_mr(struct ib_pd *pd,
  int num_phys_buf, int acc, u64 *iova_start)
  {


Why do we even bother with reg_phys_mr? Let's drop it entirely.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 26/37] IB/rdmavt: Move memory registration into rdmavt

[Christoph Hellwig]

On Thu, Dec 10, 2015 at 10:14:56AM -0500, Dennis Dalessandro wrote:
> Why? Because, it exists in qib and hfi1.
> 
> However, it seems no one is actually using this in the kernel these days and
> the core support was removed in commit 1241d7bf. Yet the function pointer
> still exists in struct ib_device. Are there plans to remove this?

Yes, I've send a patch that remove it, and which only got positive
review so far.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 26/37] IB/rdmavt: Move memory registration into rdmavt

[Dennis Dalessandro]

On Thu, Dec 10, 2015 at 04:24:44PM +0200, Sagi Grimberg wrote:




  /**
@@ -83,7 +357,31 @@ struct ib_mr *rvt_reg_phys_mr(struct ib_pd *pd,
  int num_phys_buf, int acc, u64 *iova_start)
  {


Why do we even bother with reg_phys_mr? Let's drop it entirely.


Why? Because, it exists in qib and hfi1.

However, it seems no one is actually using this in the kernel these days and 
the core support was removed in commit 1241d7bf. Yet the function pointer 
still exists in struct ib_device. Are there plans to remove this?


For this patch series I'll go ahead and remove rvt_reg_phys_mr.

-Denny
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 26/37] IB/rdmavt: Move memory registration into rdmavt

[Dennis Dalessandro]

Use the memory registration routines in hfi1 and move them to rdmavt.
A follow on patch will address removing the duplicated code in the
hfi1 and qib drivers.

Reviewed-by: Ira Weiny 
Reviewed-by: Mike Marciniszyn 
Signed-off-by: Dennis Dalessandro 
---
 drivers/infiniband/sw/rdmavt/mr.c |  710 -
 drivers/infiniband/sw/rdmavt/mr.h |   23 +
 drivers/infiniband/sw/rdmavt/vt.c |   23 +
 include/rdma/rdma_vt.h|   19 +
 4 files changed, 759 insertions(+), 16 deletions(-)

diff --git a/drivers/infiniband/sw/rdmavt/mr.c 
b/drivers/infiniband/sw/rdmavt/mr.c
index a4f4581..e809aaa 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -49,8 +49,252 @@
  */
 
 #include 
+#include 
+#include 
+#include 
+#include "vt.h"
 #include "mr.h"
 
+/*
+ * Do any intilization needed when a driver registers with rdmavt.
+ */
+int rvt_driver_mr_init(struct rvt_dev_info *rdi)
+{
+   unsigned int lkey_table_size = rdi->dparms.lkey_table_size;
+   unsigned lk_tab_size;
+   int i;
+
+   if (rdi->flags & RVT_FLAG_MR_INIT_DRIVER) {
+   rvt_pr_info(rdi, "Driver is doing MR init.\n");
+   return 0;
+   }
+
+   /*
+* The top hfi1_lkey_table_size bits are used to index the
+* table.  The lower 8 bits can be owned by the user (copied from
+* the LKEY).  The remaining bits act as a generation number or tag.
+*/
+   if (!lkey_table_size)
+   return -EINVAL;
+
+   spin_lock_init(>lk_table.lock);
+
+   rdi->lk_table.max = 1 << lkey_table_size;
+
+   /* ensure generation is at least 4 bits */
+   if (lkey_table_size > RVT_MAX_LKEY_TABLE_BITS) {
+   rvt_pr_warn(rdi, "lkey bits %u too large, reduced to %u\n",
+   lkey_table_size, RVT_MAX_LKEY_TABLE_BITS);
+   rdi->dparms.lkey_table_size = RVT_MAX_LKEY_TABLE_BITS;
+   lkey_table_size = rdi->dparms.lkey_table_size;
+   }
+   lk_tab_size = rdi->lk_table.max * sizeof(*rdi->lk_table.table);
+   rdi->lk_table.table = (struct rvt_mregion __rcu **)
+  vmalloc(lk_tab_size);
+   if (!rdi->lk_table.table)
+   return -ENOMEM;
+
+   RCU_INIT_POINTER(rdi->dma_mr, NULL);
+   for (i = 0; i < rdi->lk_table.max; i++)
+   RCU_INIT_POINTER(rdi->lk_table.table[i], NULL);
+
+   return 0;
+}
+
+/*
+ * called when drivers have unregistered or perhaps failed to register with us
+ */
+void rvt_mr_exit(struct rvt_dev_info *rdi)
+{
+   if (rdi->dma_mr)
+   rvt_pr_err(rdi, "DMA MR not null!\n");
+
+   vfree(rdi->lk_table.table);
+}
+
+static void rvt_deinit_mregion(struct rvt_mregion *mr)
+{
+   int i = mr->mapsz;
+
+   mr->mapsz = 0;
+   while (i)
+   kfree(mr->map[--i]);
+}
+
+static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
+   int count)
+{
+   int m, i = 0;
+
+   mr->mapsz = 0;
+   m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ;
+   for (; i < m; i++) {
+   mr->map[i] = kzalloc(sizeof(*mr->map[0]), GFP_KERNEL);
+   if (!mr->map[i]) {
+   rvt_deinit_mregion(mr);
+   return -ENOMEM;
+   }
+   mr->mapsz++;
+   }
+   init_completion(>comp);
+   /* count returning the ptr to user */
+   atomic_set(>refcount, 1);
+   mr->pd = pd;
+   mr->max_segs = count;
+   return 0;
+}
+
+/**
+ * rvt_alloc_lkey - allocate an lkey
+ * @mr: memory region that this lkey protects
+ * @dma_region: 0->normal key, 1->restricted DMA key
+ *
+ * Returns 0 if successful, otherwise returns -errno.
+ *
+ * Increments mr reference count as required.
+ *
+ * Sets the lkey field mr for non-dma regions.
+ *
+ */
+static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
+{
+   unsigned long flags;
+   u32 r;
+   u32 n;
+   int ret = 0;
+   struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device);
+   struct rvt_lkey_table *rkt = >lk_table;
+
+   rvt_get_mr(mr);
+   spin_lock_irqsave(>lock, flags);
+
+   /* special case for dma_mr lkey == 0 */
+   if (dma_region) {
+   struct rvt_mregion *tmr;
+
+   tmr = rcu_access_pointer(dev->dma_mr);
+   if (!tmr) {
+   rcu_assign_pointer(dev->dma_mr, mr);
+   mr->lkey_published = 1;
+   } else {
+   rvt_put_mr(mr);
+   }
+   goto success;
+   }
+
+   /* Find the next available LKEY */
+   r = rkt->next;
+   n = r;
+   for (;;) {
+   if (!rcu_access_pointer(rkt->table[r]))
+   break;
+   r = (r + 1) & (rkt->max - 1);
+   if (r == n)
+