>Or Gerlitz wrote:
>> Sean Hefty wrote:
>> We should probably skip id_priv->state == CMA_DEVICE_REMOVAL as well.
> OK
>> Is KERN_ERR what we want here?
> no, I think we can do well with warning or info level
>> Should this be goto (mutex_unlock) instead?
> yes it would be better to have it this way
Hi Sean,
Also here, I want to make sure you are ok with the final version of the patch
that changes the rdma-cm to deliver the addr change event. So this is the
version I intend to push which addresses your last comments
----
RDMA_CM_EVENT_ADDR_CHANGE event can be used by rdma-cm consumers that wish
to have their RDMA sessions always use the same links (eg <hca/port>) as the
IP stack does. In the current code, this does not happen when bonding is used
and fail-over happened, but the IB link used by an already existing session is
operating fine.
Use netevent notification for sensing that a change has happened in the IP
stack,
then scan the rdma-cm IDs list to see if there is an ID that is "misaligned" in
that respect with the IP stack, and deliver RDMA_CM_EVENT_ADDR_CHANGE for this
ID.
The user can act on the event or just ignore it
Signed-off-by: Or Gerlitz <[EMAIL PROTECTED]>
drivers/infiniband/core/cma.c | 90 ++++++++++++++++++++++++++++++++++++++++++
include/rdma/rdma_cm.h | 3 -
2 files changed, 92 insertions(+), 1 deletion(-)
Index: infiniband/drivers/infiniband/core/cma.c
===================================================================
--- infiniband.orig/drivers/infiniband/core/cma.c
+++ infiniband/drivers/infiniband/core/cma.c
@@ -168,6 +168,12 @@ struct cma_work {
struct rdma_cm_event event;
};
+struct cma_ndev_work {
+ struct work_struct work;
+ struct rdma_id_private *id;
+ struct rdma_cm_event event;
+};
+
union cma_ip_addr {
struct in6_addr ip6;
struct {
@@ -1595,6 +1601,29 @@ out:
kfree(work);
}
+static void cma_ndev_work_handler(struct work_struct *_work)
+{
+ struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work,
work);
+ struct rdma_id_private *id_priv = work->id;
+ int destroy = 0;
+
+ mutex_lock(&id_priv->handler_mutex);
+ if (id_priv->state == CMA_DESTROYING ||
+ id_priv->state == CMA_DEVICE_REMOVAL)
+ goto out;
+
+ if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
+ cma_exch(id_priv, CMA_DESTROYING);
+ destroy = 1;
+ }
+out:
+ mutex_unlock(&id_priv->handler_mutex);
+ cma_deref_id(id_priv);
+ if (destroy)
+ rdma_destroy_id(&id_priv->id);
+ kfree(work);
+}
+
static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int
timeout_ms)
{
struct rdma_route *route = &id_priv->id.route;
@@ -2723,6 +2752,64 @@ void rdma_leave_multicast(struct rdma_cm
}
EXPORT_SYMBOL(rdma_leave_multicast);
+static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private
*id_priv)
+{
+ struct rdma_dev_addr *dev_addr;
+ struct cma_ndev_work *work;
+
+ dev_addr = &id_priv->id.route.addr.dev_addr;
+
+ if (!memcmp(dev_addr->src_dev_name, ndev->name, IFNAMSIZ) &&
+ memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
+ printk(KERN_INFO "RDMA CM addr change for ndev %s used by id
%p\n",
+ ndev->name, &id_priv->id);
+ work = kzalloc(sizeof *work, GFP_KERNEL);
+ if (!work)
+ return -ENOMEM;
+ INIT_WORK(&work->work, cma_ndev_work_handler);
+ work->id = id_priv;
+ work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
+ atomic_inc(&id_priv->refcount);
+ queue_work(cma_wq, &work->work);
+ }
+
+ return 0;
+}
+
+static int cma_netdev_callback(struct notifier_block *self, unsigned long
event,
+ void *ctx)
+{
+ struct net_device *ndev = (struct net_device *)ctx;
+ struct cma_device *cma_dev;
+ struct rdma_id_private *id_priv;
+ int ret = NOTIFY_DONE;
+
+ if (dev_net(ndev) != &init_net)
+ return NOTIFY_DONE;
+
+ if (event != NETDEV_BONDING_FAILOVER)
+ return NOTIFY_DONE;
+
+ if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
+ return NOTIFY_DONE;
+
+ mutex_lock(&lock);
+ list_for_each_entry(cma_dev, &dev_list, list)
+ list_for_each_entry(id_priv, &cma_dev->id_list, list) {
+ ret = cma_netdev_change(ndev, id_priv);
+ if (ret)
+ goto out;
+ }
+
+out:
+ mutex_unlock(&lock);
+ return ret;
+}
+
+static struct notifier_block cma_nb = {
+ .notifier_call = cma_netdev_callback
+};
+
static void cma_add_one(struct ib_device *device)
{
struct cma_device *cma_dev;
@@ -2831,6 +2918,7 @@ static int cma_init(void)
ib_sa_register_client(&sa_client);
rdma_addr_register_client(&addr_client);
+ register_netdevice_notifier(&cma_nb);
ret = ib_register_client(&cma_client);
if (ret)
@@ -2838,6 +2926,7 @@ static int cma_init(void)
return 0;
err:
+ unregister_netdevice_notifier(&cma_nb);
rdma_addr_unregister_client(&addr_client);
ib_sa_unregister_client(&sa_client);
destroy_workqueue(cma_wq);
@@ -2847,6 +2936,7 @@ err:
static void cma_cleanup(void)
{
ib_unregister_client(&cma_client);
+ unregister_netdevice_notifier(&cma_nb);
rdma_addr_unregister_client(&addr_client);
ib_sa_unregister_client(&sa_client);
destroy_workqueue(cma_wq);
Index: infiniband/include/rdma/rdma_cm.h
===================================================================
--- infiniband.orig/include/rdma/rdma_cm.h
+++ infiniband/include/rdma/rdma_cm.h
@@ -57,7 +57,8 @@ enum rdma_cm_event_type {
RDMA_CM_EVENT_DISCONNECTED,
RDMA_CM_EVENT_DEVICE_REMOVAL,
RDMA_CM_EVENT_MULTICAST_JOIN,
- RDMA_CM_EVENT_MULTICAST_ERROR
+ RDMA_CM_EVENT_MULTICAST_ERROR,
+ RDMA_CM_EVENT_ADDR_CHANGE
};
enum rdma_port_space {
_______________________________________________
general mailing list
[email protected]
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
