[PATCH] staging: lustre: o2iblnd: Enable Multiple OPA Endpoints between Nodes

[Doug Oucharek]

OPA driver optimizations are based on the MPI model where it is
expected to have multiple endpoints between two given nodes. To
enable this optimization for Lustre, we need to make it possible,
via an LND-specific tuneable, to create multiple endpoints and to
balance the traffic over them.

Both sides of a connection must have this patch for it to work.
Only the active side of the connection (usually the client)
needs to have the new tuneable set > 1.

Signed-off-by: Doug Oucharek 
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-8943
Reviewed-on: https://review.whamcloud.com/25168
Reviewed-by: Amir Shehata 
Reviewed-by: Dmitry Eremin 
Reviewed-by: James Simmons 
Reviewed-by: Oleg Drokin 
Signed-off-by: Doug Oucharek 
---
 .../lustre/include/uapi/linux/lnet/lnet-dlc.h  |  3 ++-
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h| 17 ---
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 25 +++---
 .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |  9 
 4 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h 
b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
index e45d828..c1619f4 100644
--- a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
@@ -53,7 +53,8 @@ struct lnet_ioctl_config_o2iblnd_tunables {
__u32 lnd_fmr_pool_size;
__u32 lnd_fmr_flush_trigger;
__u32 lnd_fmr_cache;
-   __u32 pad;
+   __u16 lnd_conns_per_peer;
+   __u16 pad;
 };
 
 struct lnet_ioctl_config_lnd_tunables {
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index ca6e09d..da8929d 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -568,6 +568,8 @@ struct kib_peer {
lnet_nid_t   ibp_nid; /* who's on the other end(s) */
struct lnet_ni  *ibp_ni; /* LNet interface */
struct list_head ibp_conns;   /* all active connections */
+   struct kib_conn *ibp_next_conn;  /* next connection to send on for
+   round robin */
struct list_head ibp_tx_queue;/* msgs waiting for a conn */
__u64ibp_incarnation; /* incarnation of peer */
/* when (in jiffies) I was last alive */
@@ -581,7 +583,7 @@ struct kib_peer {
/* current active connection attempts */
unsigned short  ibp_connecting;
/* reconnect this peer later */
-   unsigned short  ibp_reconnecting:1;
+   unsigned char   ibp_reconnecting;
/* counter of how many times we triggered a conn race */
unsigned char   ibp_races;
/* # consecutive reconnection attempts to this peer */
@@ -744,10 +746,19 @@ struct kib_peer {
 static inline struct kib_conn *
 kiblnd_get_conn_locked(struct kib_peer *peer)
 {
+   struct list_head *next;
+
LASSERT(!list_empty(>ibp_conns));
 
-   /* just return the first connection */
-   return list_entry(peer->ibp_conns.next, struct kib_conn, ibc_list);
+   /* Advance to next connection, be sure to skip the head node */
+   if (!peer->ibp_next_conn ||
+   peer->ibp_next_conn->ibc_list.next == >ibp_conns)
+   next = peer->ibp_conns.next;
+   else
+   next = peer->ibp_next_conn->ibc_list.next;
+   peer->ibp_next_conn = list_entry(next, struct kib_conn, ibc_list);
+
+   return peer->ibp_next_conn;
 }
 
 static inline int
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index b4a182d..303e4e1 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -1250,7 +1250,6 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
 
LASSERT(net);
LASSERT(peer->ibp_connecting > 0);
-   LASSERT(!peer->ibp_reconnecting);
 
cmid = kiblnd_rdma_create_id(kiblnd_cm_callback, peer, RDMA_PS_TCP,
 IB_QPT_RC);
@@ -1332,7 +1331,7 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
 
LASSERT(!peer->ibp_accepting && !peer->ibp_connecting &&
list_empty(>ibp_conns));
-   peer->ibp_reconnecting = 0;
+   peer->ibp_reconnecting--;
 
if (!kiblnd_peer_active(peer)) {
list_splice_init(>ibp_tx_queue, );
@@ -1365,6 +1364,8 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
rwlock_t *g_lock = _data.kib_global_lock;
unsigned long flags;
int rc;
+   inti;
+   struct lnet_ioctl_config_o2iblnd_tunables 

[PATCH] staging: lustre: o2iblnd: Enable Multiple OPA Endpoints between Nodes

[Doug Oucharek]

OPA driver optimizations are based on the MPI model where it is
expected to have multiple endpoints between two given nodes. To
enable this optimization for Lustre, we need to make it possible,
via an LND-specific tuneable, to create multiple endpoints and to
balance the traffic over them.

Both sides of a connection must have this patch for it to work.
Only the active side of the connection (usually the client)
needs to have the new tuneable set > 1.

Signed-off-by: Doug Oucharek 
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-8943
Reviewed-on: https://review.whamcloud.com/25168
Reviewed-by: Amir Shehata 
Reviewed-by: Dmitry Eremin 
Reviewed-by: James Simmons 
Reviewed-by: Oleg Drokin 
Signed-off-by: Doug Oucharek 
---
 .../lustre/include/uapi/linux/lnet/lnet-dlc.h  |  3 ++-
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h| 17 ---
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 25 +++---
 .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |  9 
 4 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h 
b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
index e45d828..c1619f4 100644
--- a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
@@ -53,7 +53,8 @@ struct lnet_ioctl_config_o2iblnd_tunables {
__u32 lnd_fmr_pool_size;
__u32 lnd_fmr_flush_trigger;
__u32 lnd_fmr_cache;
-   __u32 pad;
+   __u16 lnd_conns_per_peer;
+   __u16 pad;
 };
 
 struct lnet_ioctl_config_lnd_tunables {
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index ca6e09d..da8929d 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -568,6 +568,8 @@ struct kib_peer {
lnet_nid_t   ibp_nid; /* who's on the other end(s) */
struct lnet_ni  *ibp_ni; /* LNet interface */
struct list_head ibp_conns;   /* all active connections */
+   struct kib_conn *ibp_next_conn;  /* next connection to send on for
+   round robin */
struct list_head ibp_tx_queue;/* msgs waiting for a conn */
__u64ibp_incarnation; /* incarnation of peer */
/* when (in jiffies) I was last alive */
@@ -581,7 +583,7 @@ struct kib_peer {
/* current active connection attempts */
unsigned short  ibp_connecting;
/* reconnect this peer later */
-   unsigned short  ibp_reconnecting:1;
+   unsigned char   ibp_reconnecting;
/* counter of how many times we triggered a conn race */
unsigned char   ibp_races;
/* # consecutive reconnection attempts to this peer */
@@ -744,10 +746,19 @@ struct kib_peer {
 static inline struct kib_conn *
 kiblnd_get_conn_locked(struct kib_peer *peer)
 {
+   struct list_head *next;
+
LASSERT(!list_empty(>ibp_conns));
 
-   /* just return the first connection */
-   return list_entry(peer->ibp_conns.next, struct kib_conn, ibc_list);
+   /* Advance to next connection, be sure to skip the head node */
+   if (!peer->ibp_next_conn ||
+   peer->ibp_next_conn->ibc_list.next == >ibp_conns)
+   next = peer->ibp_conns.next;
+   else
+   next = peer->ibp_next_conn->ibc_list.next;
+   peer->ibp_next_conn = list_entry(next, struct kib_conn, ibc_list);
+
+   return peer->ibp_next_conn;
 }
 
 static inline int
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index b4a182d..303e4e1 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -1250,7 +1250,6 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
 
LASSERT(net);
LASSERT(peer->ibp_connecting > 0);
-   LASSERT(!peer->ibp_reconnecting);
 
cmid = kiblnd_rdma_create_id(kiblnd_cm_callback, peer, RDMA_PS_TCP,
 IB_QPT_RC);
@@ -1332,7 +1331,7 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
 
LASSERT(!peer->ibp_accepting && !peer->ibp_connecting &&
list_empty(>ibp_conns));
-   peer->ibp_reconnecting = 0;
+   peer->ibp_reconnecting--;
 
if (!kiblnd_peer_active(peer)) {
list_splice_init(>ibp_tx_queue, );
@@ -1365,6 +1364,8 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
rwlock_t *g_lock = _data.kib_global_lock;
unsigned long flags;
int rc;
+   inti;
+   struct lnet_ioctl_config_o2iblnd_tunables *tunables;
 
/*
 * If I get here, I've committed to send, so I complete the tx with
@@ -1461,7 +1462,8 @@ static int 

Re: [PATCH] staging: lustre: o2iblnd: Enable Multiple OPA Endpoints between Nodes

[Greg Kroah-Hartman]

On Thu, Mar 15, 2018 at 02:53:46PM -0700, Doug Oucharek wrote:
> OPA driver optimizations are based on the MPI model where it is
> expected to have multiple endpoints between two given nodes. To
> enable this optimization for Lustre, we need to make it possible,
> via an LND-specific tuneable, to create multiple endpoints and to
> balance the traffic over them.
> 
> Both sides of a connection must have this patch for it to work.
> Only the active side of the connection (usually the client)
> needs to have the new tuneable set > 1.
> 
> Signed-off-by: Doug Oucharek 
> Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-8943
> Reviewed-on: https://review.whamcloud.com/25168
> Reviewed-by: Amir Shehata 
> Reviewed-by: Dmitry Eremin 
> Reviewed-by: James Simmons 
> Reviewed-by: Oleg Drokin 
> Signed-off-by: Doug Oucharek 
> ---
> .../lustre/include/uapi/linux/lnet/lnet-dlc.h  |  3 ++-
> .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h| 17 ---
> .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 25 +++---
> .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |  9 
> 4 files changed, 42 insertions(+), 12 deletions(-)

Does not apply to my tree at all :(

Re: [PATCH] staging: lustre: o2iblnd: Enable Multiple OPA Endpoints between Nodes

[Greg Kroah-Hartman]

On Thu, Mar 15, 2018 at 02:53:46PM -0700, Doug Oucharek wrote:
> OPA driver optimizations are based on the MPI model where it is
> expected to have multiple endpoints between two given nodes. To
> enable this optimization for Lustre, we need to make it possible,
> via an LND-specific tuneable, to create multiple endpoints and to
> balance the traffic over them.
> 
> Both sides of a connection must have this patch for it to work.
> Only the active side of the connection (usually the client)
> needs to have the new tuneable set > 1.
> 
> Signed-off-by: Doug Oucharek 
> Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-8943
> Reviewed-on: https://review.whamcloud.com/25168
> Reviewed-by: Amir Shehata 
> Reviewed-by: Dmitry Eremin 
> Reviewed-by: James Simmons 
> Reviewed-by: Oleg Drokin 
> Signed-off-by: Doug Oucharek 
> ---
> .../lustre/include/uapi/linux/lnet/lnet-dlc.h  |  3 ++-
> .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h| 17 ---
> .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 25 +++---
> .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |  9 
> 4 files changed, 42 insertions(+), 12 deletions(-)

Does not apply to my tree at all :(

[PATCH] staging: lustre: o2iblnd: Enable Multiple OPA Endpoints between Nodes

[Doug Oucharek]

OPA driver optimizations are based on the MPI model where it is
expected to have multiple endpoints between two given nodes. To
enable this optimization for Lustre, we need to make it possible,
via an LND-specific tuneable, to create multiple endpoints and to
balance the traffic over them.

Both sides of a connection must have this patch for it to work.
Only the active side of the connection (usually the client)
needs to have the new tuneable set > 1.

Signed-off-by: Doug Oucharek 
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-8943
Reviewed-on: https://review.whamcloud.com/25168
Reviewed-by: Amir Shehata 
Reviewed-by: Dmitry Eremin 
Reviewed-by: James Simmons 
Reviewed-by: Oleg Drokin 
Signed-off-by: Doug Oucharek 
---
.../lustre/include/uapi/linux/lnet/lnet-dlc.h  |  3 ++-
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h| 17 ---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 25 +++---
.../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |  9 
4 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h 
b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
index e45d828..c1619f4 100644
--- a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
@@ -53,7 +53,8 @@ struct lnet_ioctl_config_o2iblnd_tunables {
__u32 lnd_fmr_pool_size;
__u32 lnd_fmr_flush_trigger;
__u32 lnd_fmr_cache;
-   __u32 pad;
+   __u16 lnd_conns_per_peer;
+   __u16 pad;
};

struct lnet_ioctl_config_lnd_tunables {
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index b18911d..af5f573 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -568,6 +568,8 @@ struct kib_peer {
lnet_nid_t   ibp_nid; /* who's on the other end(s) */
struct lnet_ni  *ibp_ni; /* LNet interface */
struct list_head ibp_conns;   /* all active connections */
+   struct kib_conn *ibp_next_conn;  /* next connection to send on for
+   round robin */
struct list_head ibp_tx_queue;/* msgs waiting for a conn */
__u64ibp_incarnation; /* incarnation of peer */
/* when (in jiffies) I was last alive */
@@ -581,7 +583,7 @@ struct kib_peer {
/* current active connection attempts */
unsigned short  ibp_connecting;
/* reconnect this peer later */
-   unsigned short  ibp_reconnecting:1;
+   unsigned char   ibp_reconnecting;
/* counter of how many times we triggered a conn race */
unsigned char   ibp_races;
/* # consecutive reconnection attempts to this peer */
@@ -744,10 +746,19 @@ struct kib_peer {
static inline struct kib_conn *
kiblnd_get_conn_locked(struct kib_peer *peer)
{
+   struct list_head *next;
+
LASSERT(!list_empty(>ibp_conns));

-   /* just return the first connection */
-   return list_entry(peer->ibp_conns.next, struct kib_conn, ibc_list);
+   /* Advance to next connection, be sure to skip the head node */
+   if (!peer->ibp_next_conn ||
+   peer->ibp_next_conn->ibc_list.next == >ibp_conns)
+   next = peer->ibp_conns.next;
+   else
+   next = peer->ibp_next_conn->ibc_list.next;
+   peer->ibp_next_conn = list_entry(next, struct kib_conn, ibc_list);
+
+   return peer->ibp_next_conn;
}

static inline int
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 6690a6c..3cb1f720 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -1250,7 +1250,6 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,

LASSERT(net);
LASSERT(peer->ibp_connecting > 0);
-   LASSERT(!peer->ibp_reconnecting);

cmid = kiblnd_rdma_create_id(kiblnd_cm_callback, peer, RDMA_PS_TCP,
 IB_QPT_RC);
@@ -1332,7 +1331,7 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,

LASSERT(!peer->ibp_accepting && !peer->ibp_connecting &&
list_empty(>ibp_conns));
-   peer->ibp_reconnecting = 0;
+   peer->ibp_reconnecting--;

if (!kiblnd_peer_active(peer)) {
list_splice_init(>ibp_tx_queue, );
@@ -1365,6 +1364,8 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
rwlock_t *g_lock = _data.kib_global_lock;
unsigned long flags;
int rc;
+   inti;
+   struct lnet_ioctl_config_o2iblnd_tunables *tunables;

[PATCH] staging: lustre: o2iblnd: Enable Multiple OPA Endpoints between Nodes

[Doug Oucharek]

OPA driver optimizations are based on the MPI model where it is
expected to have multiple endpoints between two given nodes. To
enable this optimization for Lustre, we need to make it possible,
via an LND-specific tuneable, to create multiple endpoints and to
balance the traffic over them.

Both sides of a connection must have this patch for it to work.
Only the active side of the connection (usually the client)
needs to have the new tuneable set > 1.

Signed-off-by: Doug Oucharek 
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-8943
Reviewed-on: https://review.whamcloud.com/25168
Reviewed-by: Amir Shehata 
Reviewed-by: Dmitry Eremin 
Reviewed-by: James Simmons 
Reviewed-by: Oleg Drokin 
Signed-off-by: Doug Oucharek 
---
.../lustre/include/uapi/linux/lnet/lnet-dlc.h  |  3 ++-
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h| 17 ---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 25 +++---
.../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |  9 
4 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h 
b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
index e45d828..c1619f4 100644
--- a/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
@@ -53,7 +53,8 @@ struct lnet_ioctl_config_o2iblnd_tunables {
__u32 lnd_fmr_pool_size;
__u32 lnd_fmr_flush_trigger;
__u32 lnd_fmr_cache;
-   __u32 pad;
+   __u16 lnd_conns_per_peer;
+   __u16 pad;
};

struct lnet_ioctl_config_lnd_tunables {
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index b18911d..af5f573 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -568,6 +568,8 @@ struct kib_peer {
lnet_nid_t   ibp_nid; /* who's on the other end(s) */
struct lnet_ni  *ibp_ni; /* LNet interface */
struct list_head ibp_conns;   /* all active connections */
+   struct kib_conn *ibp_next_conn;  /* next connection to send on for
+   round robin */
struct list_head ibp_tx_queue;/* msgs waiting for a conn */
__u64ibp_incarnation; /* incarnation of peer */
/* when (in jiffies) I was last alive */
@@ -581,7 +583,7 @@ struct kib_peer {
/* current active connection attempts */
unsigned short  ibp_connecting;
/* reconnect this peer later */
-   unsigned short  ibp_reconnecting:1;
+   unsigned char   ibp_reconnecting;
/* counter of how many times we triggered a conn race */
unsigned char   ibp_races;
/* # consecutive reconnection attempts to this peer */
@@ -744,10 +746,19 @@ struct kib_peer {
static inline struct kib_conn *
kiblnd_get_conn_locked(struct kib_peer *peer)
{
+   struct list_head *next;
+
LASSERT(!list_empty(>ibp_conns));

-   /* just return the first connection */
-   return list_entry(peer->ibp_conns.next, struct kib_conn, ibc_list);
+   /* Advance to next connection, be sure to skip the head node */
+   if (!peer->ibp_next_conn ||
+   peer->ibp_next_conn->ibc_list.next == >ibp_conns)
+   next = peer->ibp_conns.next;
+   else
+   next = peer->ibp_next_conn->ibc_list.next;
+   peer->ibp_next_conn = list_entry(next, struct kib_conn, ibc_list);
+
+   return peer->ibp_next_conn;
}

static inline int
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c 
b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 6690a6c..3cb1f720 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -1250,7 +1250,6 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,

LASSERT(net);
LASSERT(peer->ibp_connecting > 0);
-   LASSERT(!peer->ibp_reconnecting);

cmid = kiblnd_rdma_create_id(kiblnd_cm_callback, peer, RDMA_PS_TCP,
 IB_QPT_RC);
@@ -1332,7 +1331,7 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,

LASSERT(!peer->ibp_accepting && !peer->ibp_connecting &&
list_empty(>ibp_conns));
-   peer->ibp_reconnecting = 0;
+   peer->ibp_reconnecting--;

if (!kiblnd_peer_active(peer)) {
list_splice_init(>ibp_tx_queue, );
@@ -1365,6 +1364,8 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
rwlock_t *g_lock = _data.kib_global_lock;
unsigned long flags;
int rc;
+   inti;
+   struct lnet_ioctl_config_o2iblnd_tunables *tunables;

/*
 * If I get here, I've committed to send, so I complete the tx with
@@ -1461,7 +1462,8 @@ static int