[PATCH] svc_rdma: use local_dma_lkey

2015-12-22 Thread Christoph Hellwig 
We now alwasy have a per-PD local_dma_lkey available.  Make use of that
fact in svc_rdma and stop registering our own MR.

Signed-off-by: Christoph Hellwig 
Reviewed-by: Sagi Grimberg 
Reviewed-by: Jason Gunthorpe 
Reviewed-by: Chuck Lever 
---
 include/linux/sunrpc/svc_rdma.h|  2 --
 net/sunrpc/xprtrdma/svc_rdma_backchannel.c |  2 +-
 net/sunrpc/xprtrdma/svc_rdma_recvfrom.c|  4 ++--
 net/sunrpc/xprtrdma/svc_rdma_sendto.c  |  6 ++---
 net/sunrpc/xprtrdma/svc_rdma_transport.c   | 36 --
 5 files changed, 10 insertions(+), 40 deletions(-)

diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index b13513a..5322fea 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -156,13 +156,11 @@ struct svcxprt_rdma {
struct ib_qp *sc_qp;
struct ib_cq *sc_rq_cq;
struct ib_cq *sc_sq_cq;
-   struct ib_mr *sc_phys_mr;   /* MR for server memory */
int  (*sc_reader)(struct svcxprt_rdma *,
  struct svc_rqst *,
  struct svc_rdma_op_ctxt *,
  int *, u32 *, u32, u32, u64, bool);
u32  sc_dev_caps;   /* distilled device caps */
-   u32  sc_dma_lkey;   /* local dma key */
unsigned int sc_frmr_pg_list_len;
struct list_head sc_frmr_q;
spinlock_t   sc_frmr_q_lock;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c 
b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
index 417cec1..c428734 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
@@ -128,7 +128,7 @@ static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma,
 
ctxt->wr_op = IB_WR_SEND;
ctxt->direction = DMA_TO_DEVICE;
-   ctxt->sge[0].lkey = rdma->sc_dma_lkey;
+   ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey;
ctxt->sge[0].length = sndbuf->len;
ctxt->sge[0].addr =
ib_dma_map_page(rdma->sc_cm_id->device, ctxt->pages[0], 0,
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c 
b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index 3dfe464..c8b8a8b 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -144,6 +144,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
 
head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];
head->arg.page_len += len;
+
head->arg.len += len;
if (!pg_off)
head->count++;
@@ -160,8 +161,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
goto err;
atomic_inc(>sc_dma_used);
 
-   /* The lkey here is either a local dma lkey or a dma_mr lkey */
-   ctxt->sge[pno].lkey = xprt->sc_dma_lkey;
+   ctxt->sge[pno].lkey = xprt->sc_pd->local_dma_lkey;
ctxt->sge[pno].length = len;
ctxt->count++;
 
diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c 
b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
index ced3151..20bd5d4 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
@@ -265,7 +265,7 @@ static int send_write(struct svcxprt_rdma *xprt, struct 
svc_rqst *rqstp,
 sge[sge_no].addr))
goto err;
atomic_inc(>sc_dma_used);
-   sge[sge_no].lkey = xprt->sc_dma_lkey;
+   sge[sge_no].lkey = xprt->sc_pd->local_dma_lkey;
ctxt->count++;
sge_off = 0;
sge_no++;
@@ -487,7 +487,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
ctxt->count = 1;
 
/* Prepare the SGE for the RPCRDMA Header */
-   ctxt->sge[0].lkey = rdma->sc_dma_lkey;
+   ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey;
ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp);
ctxt->sge[0].addr =
ib_dma_map_page(rdma->sc_cm_id->device, page, 0,
@@ -511,7 +511,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
 ctxt->sge[sge_no].addr))
goto err;
atomic_inc(>sc_dma_used);
-   ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey;
+   ctxt->sge[sge_no].lkey = rdma->sc_pd->local_dma_lkey;
ctxt->sge[sge_no].length = sge_bytes;
}
if (byte_count != 0) {
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c 
b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index abfbd02..faf4c49 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -232,11 +232,11 @@ void svc_rdma_unmap_dma(struct

Re: [PATCH] svc_rdma: use local_dma_lkey

2015-12-16 Thread Jason Gunthorpe 
On Wed, Dec 16, 2015 at 04:11:04PM +0100, Christoph Hellwig wrote:
> We now alwasy have a per-PD local_dma_lkey available.  Make use of that
> fact in svc_rdma and stop registering our own MR.
> 
> Signed-off-by: Christoph Hellwig 

Reviewed-by: Jason Gunthorpe 

> +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
> @@ -144,6 +144,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
>  
>   head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];
>   head->arg.page_len += len;
> +
>   head->arg.len += len;
>   if (!pg_off)
>   head->count++;

Was this hunk deliberate?

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH] svc_rdma: use local_dma_lkey

2015-12-16 Thread Christoph Hellwig 
We now alwasy have a per-PD local_dma_lkey available.  Make use of that
fact in svc_rdma and stop registering our own MR.

Signed-off-by: Christoph Hellwig 
---
 include/linux/sunrpc/svc_rdma.h|  2 --
 net/sunrpc/xprtrdma/svc_rdma_backchannel.c |  2 +-
 net/sunrpc/xprtrdma/svc_rdma_recvfrom.c|  4 ++--
 net/sunrpc/xprtrdma/svc_rdma_sendto.c  |  6 ++---
 net/sunrpc/xprtrdma/svc_rdma_transport.c   | 36 --
 5 files changed, 10 insertions(+), 40 deletions(-)

diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index b13513a..5322fea 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -156,13 +156,11 @@ struct svcxprt_rdma {
struct ib_qp *sc_qp;
struct ib_cq *sc_rq_cq;
struct ib_cq *sc_sq_cq;
-   struct ib_mr *sc_phys_mr;   /* MR for server memory */
int  (*sc_reader)(struct svcxprt_rdma *,
  struct svc_rqst *,
  struct svc_rdma_op_ctxt *,
  int *, u32 *, u32, u32, u64, bool);
u32  sc_dev_caps;   /* distilled device caps */
-   u32  sc_dma_lkey;   /* local dma key */
unsigned int sc_frmr_pg_list_len;
struct list_head sc_frmr_q;
spinlock_t   sc_frmr_q_lock;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c 
b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
index 417cec1..c428734 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
@@ -128,7 +128,7 @@ static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma,
 
ctxt->wr_op = IB_WR_SEND;
ctxt->direction = DMA_TO_DEVICE;
-   ctxt->sge[0].lkey = rdma->sc_dma_lkey;
+   ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey;
ctxt->sge[0].length = sndbuf->len;
ctxt->sge[0].addr =
ib_dma_map_page(rdma->sc_cm_id->device, ctxt->pages[0], 0,
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c 
b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index 3dfe464..c8b8a8b 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -144,6 +144,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
 
head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];
head->arg.page_len += len;
+
head->arg.len += len;
if (!pg_off)
head->count++;
@@ -160,8 +161,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
goto err;
atomic_inc(>sc_dma_used);
 
-   /* The lkey here is either a local dma lkey or a dma_mr lkey */
-   ctxt->sge[pno].lkey = xprt->sc_dma_lkey;
+   ctxt->sge[pno].lkey = xprt->sc_pd->local_dma_lkey;
ctxt->sge[pno].length = len;
ctxt->count++;
 
diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c 
b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
index ced3151..20bd5d4 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
@@ -265,7 +265,7 @@ static int send_write(struct svcxprt_rdma *xprt, struct 
svc_rqst *rqstp,
 sge[sge_no].addr))
goto err;
atomic_inc(>sc_dma_used);
-   sge[sge_no].lkey = xprt->sc_dma_lkey;
+   sge[sge_no].lkey = xprt->sc_pd->local_dma_lkey;
ctxt->count++;
sge_off = 0;
sge_no++;
@@ -487,7 +487,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
ctxt->count = 1;
 
/* Prepare the SGE for the RPCRDMA Header */
-   ctxt->sge[0].lkey = rdma->sc_dma_lkey;
+   ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey;
ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp);
ctxt->sge[0].addr =
ib_dma_map_page(rdma->sc_cm_id->device, page, 0,
@@ -511,7 +511,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
 ctxt->sge[sge_no].addr))
goto err;
atomic_inc(>sc_dma_used);
-   ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey;
+   ctxt->sge[sge_no].lkey = rdma->sc_pd->local_dma_lkey;
ctxt->sge[sge_no].length = sge_bytes;
}
if (byte_count != 0) {
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c 
b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index abfbd02..faf4c49 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -232,11 +232,11 @@ void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt)
for (i = 0; i < ctxt->count && ctxt->sge[i].length; i++) {
/*
 * Unmap the DMA addr in the SGE if

Re: [PATCH] svc_rdma: use local_dma_lkey

2015-12-16 Thread Chuck Lever 

> On Dec 16, 2015, at 10:11 AM, Christoph Hellwig  wrote:
> 
> We now alwasy have a per-PD local_dma_lkey available.  Make use of that
> fact in svc_rdma and stop registering our own MR.
> 
> Signed-off-by: Christoph Hellwig 

Reviewed-by: Chuck Lever 

> ---
> include/linux/sunrpc/svc_rdma.h|  2 --
> net/sunrpc/xprtrdma/svc_rdma_backchannel.c |  2 +-
> net/sunrpc/xprtrdma/svc_rdma_recvfrom.c|  4 ++--
> net/sunrpc/xprtrdma/svc_rdma_sendto.c  |  6 ++---
> net/sunrpc/xprtrdma/svc_rdma_transport.c   | 36 --
> 5 files changed, 10 insertions(+), 40 deletions(-)
> 
> diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
> index b13513a..5322fea 100644
> --- a/include/linux/sunrpc/svc_rdma.h
> +++ b/include/linux/sunrpc/svc_rdma.h
> @@ -156,13 +156,11 @@ struct svcxprt_rdma {
>   struct ib_qp *sc_qp;
>   struct ib_cq *sc_rq_cq;
>   struct ib_cq *sc_sq_cq;
> - struct ib_mr *sc_phys_mr;   /* MR for server memory */
>   int  (*sc_reader)(struct svcxprt_rdma *,
> struct svc_rqst *,
> struct svc_rdma_op_ctxt *,
> int *, u32 *, u32, u32, u64, bool);
>   u32  sc_dev_caps;   /* distilled device caps */
> - u32  sc_dma_lkey;   /* local dma key */
>   unsigned int sc_frmr_pg_list_len;
>   struct list_head sc_frmr_q;
>   spinlock_t   sc_frmr_q_lock;
> diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c 
> b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
> index 417cec1..c428734 100644
> --- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
> +++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
> @@ -128,7 +128,7 @@ static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma,
> 
>   ctxt->wr_op = IB_WR_SEND;
>   ctxt->direction = DMA_TO_DEVICE;
> - ctxt->sge[0].lkey = rdma->sc_dma_lkey;
> + ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey;
>   ctxt->sge[0].length = sndbuf->len;
>   ctxt->sge[0].addr =
>   ib_dma_map_page(rdma->sc_cm_id->device, ctxt->pages[0], 0,
> diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c 
> b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
> index 3dfe464..c8b8a8b 100644
> --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
> +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
> @@ -144,6 +144,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
> 
>   head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];
>   head->arg.page_len += len;
> +
>   head->arg.len += len;
>   if (!pg_off)
>   head->count++;
> @@ -160,8 +161,7 @@ int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt,
>   goto err;
>   atomic_inc(>sc_dma_used);
> 
> - /* The lkey here is either a local dma lkey or a dma_mr lkey */
> - ctxt->sge[pno].lkey = xprt->sc_dma_lkey;
> + ctxt->sge[pno].lkey = xprt->sc_pd->local_dma_lkey;
>   ctxt->sge[pno].length = len;
>   ctxt->count++;
> 
> diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c 
> b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
> index ced3151..20bd5d4 100644
> --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c
> +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c
> @@ -265,7 +265,7 @@ static int send_write(struct svcxprt_rdma *xprt, struct 
> svc_rqst *rqstp,
>sge[sge_no].addr))
>   goto err;
>   atomic_inc(>sc_dma_used);
> - sge[sge_no].lkey = xprt->sc_dma_lkey;
> + sge[sge_no].lkey = xprt->sc_pd->local_dma_lkey;
>   ctxt->count++;
>   sge_off = 0;
>   sge_no++;
> @@ -487,7 +487,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
>   ctxt->count = 1;
> 
>   /* Prepare the SGE for the RPCRDMA Header */
> - ctxt->sge[0].lkey = rdma->sc_dma_lkey;
> + ctxt->sge[0].lkey = rdma->sc_pd->local_dma_lkey;
>   ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp);
>   ctxt->sge[0].addr =
>   ib_dma_map_page(rdma->sc_cm_id->device, page, 0,
> @@ -511,7 +511,7 @@ static int send_reply(struct svcxprt_rdma *rdma,
>ctxt->sge[sge_no].addr))
>   goto err;
>   atomic_inc(>sc_dma_used);
> - ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey;
> + ctxt->sge[sge_no].lkey = rdma->sc_pd->local_dma_lkey;
>   ctxt->sge[sge_no].length = sge_bytes;
>   }
>   if (byte_count != 0) {
> diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c 
> b/net/sunrpc/xprtrdma/svc_rdma_transport.c
> index abfbd02..faf4c49 100644
> --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
> +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c

Re: [PATCH] svc_rdma: use local_dma_lkey

2015-12-16 Thread Sagi Grimberg 

Looks good,

Reviewed-by: Sagi Grimberg 
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html