----- Forwarded message from Eli Cohen <[email protected]> ----- Date: Thu, 25 Jun 2009 17:59:00 +0300 From: Eli Cohen <[email protected]> User-Agent: Mutt/1.5.18 (2008-05-17) To: Sean Hefty <[email protected]>, Roland Dreier <[email protected]> Subject: [PATCH 4/9 v2] ib_core: Add RDMAoE SA support
Add support for resolving paths and joining multicast group for RDMAoE ports. The Ethernet specific code will complete immediately but will call the callback from a workqueue context to avoid deadloks. Ethernet code is implemented in a new file, rdmaoe_sa.c, and a header file was also added to expose the new API it supports. Consumers who want to use this API need to be changed. Signed-off-by: Eli Cohen <[email protected]> --- I changed this patch such that RDMAoE functionality is put in it own file. Some struct definitions were moved to sa.h and to a new multicast.h file. This way, there are less changes to core SA files. Let me know what you think of this approach. drivers/infiniband/core/Makefile | 2 +- drivers/infiniband/core/multicast.c | 43 +-------- drivers/infiniband/core/multicast.h | 79 ++++++++++++++ drivers/infiniband/core/rdmaoe_sa.c | 194 +++++++++++++++++++++++++++++++++++ drivers/infiniband/core/sa.h | 24 +++++ drivers/infiniband/core/sa_query.c | 26 +---- include/rdma/rdmaoe_sa.h | 61 +++++++++++ 7 files changed, 364 insertions(+), 65 deletions(-) create mode 100644 drivers/infiniband/core/multicast.h create mode 100644 drivers/infiniband/core/rdmaoe_sa.c create mode 100644 include/rdma/rdmaoe_sa.h diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index cb1ab3e..88d1c23 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -13,7 +13,7 @@ ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o ib_mad-y := mad.o smi.o agent.o mad_rmpp.o -ib_sa-y := sa_query.o multicast.o +ib_sa-y := sa_query.o multicast.o rdmaoe_sa.o ib_cm-y := cm.o diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c index 107f170..727a55a 100644 --- a/drivers/infiniband/core/multicast.c +++ b/drivers/infiniband/core/multicast.c @@ -39,6 +39,7 @@ #include <rdma/ib_cache.h> #include "sa.h" +#include "multicast.h" static void mcast_add_one(struct ib_device *device); static void mcast_remove_one(struct ib_device *device); @@ -72,52 +73,10 @@ struct mcast_device { struct mcast_port port[0]; }; -enum mcast_state { - MCAST_JOINING, - MCAST_MEMBER, - MCAST_ERROR, -}; - -enum mcast_group_state { - MCAST_IDLE, - MCAST_BUSY, - MCAST_GROUP_ERROR, - MCAST_PKEY_EVENT -}; - enum { MCAST_INVALID_PKEY_INDEX = 0xFFFF }; -struct mcast_member; - -struct mcast_group { - struct ib_sa_mcmember_rec rec; - struct rb_node node; - struct mcast_port *port; - spinlock_t lock; - struct work_struct work; - struct list_head pending_list; - struct list_head active_list; - struct mcast_member *last_join; - int members[3]; - atomic_t refcount; - enum mcast_group_state state; - struct ib_sa_query *query; - int query_id; - u16 pkey_index; -}; - -struct mcast_member { - struct ib_sa_multicast multicast; - struct ib_sa_client *client; - struct mcast_group *group; - struct list_head list; - enum mcast_state state; - atomic_t refcount; - struct completion comp; -}; - static void join_handler(int status, struct ib_sa_mcmember_rec *rec, void *context); static void leave_handler(int status, struct ib_sa_mcmember_rec *rec, diff --git a/drivers/infiniband/core/multicast.h b/drivers/infiniband/core/multicast.h new file mode 100644 index 0000000..17eb9fe --- /dev/null +++ b/drivers/infiniband/core/multicast.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2009 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef MULTICAST_H +#define MULTICAST_H + +enum mcast_state { + MCAST_JOINING, + MCAST_MEMBER, + MCAST_ERROR, +}; + +enum mcast_group_state { + MCAST_IDLE, + MCAST_BUSY, + MCAST_GROUP_ERROR, + MCAST_PKEY_EVENT +}; + +struct mcast_member; + +struct mcast_group { + struct ib_sa_mcmember_rec rec; + struct rb_node node; + struct mcast_port *port; + spinlock_t lock; + struct work_struct work; + struct list_head pending_list; + struct list_head active_list; + struct mcast_member *last_join; + int members[3]; + atomic_t refcount; + enum mcast_group_state state; + struct ib_sa_query *query; + int query_id; + u16 pkey_index; +}; + +struct mcast_member { + struct ib_sa_multicast multicast; + struct ib_sa_client *client; + struct mcast_group *group; + struct list_head list; + enum mcast_state state; + atomic_t refcount; + struct completion comp; +}; + +#endif /* MULTICAST_H */ + diff --git a/drivers/infiniband/core/rdmaoe_sa.c b/drivers/infiniband/core/rdmaoe_sa.c new file mode 100644 index 0000000..f751f81 --- /dev/null +++ b/drivers/infiniband/core/rdmaoe_sa.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2009 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/workqueue.h> +#include <rdma/ib_sa.h> +#include <rdma/rdmaoe_sa.h> +#include "sa.h" +#include "multicast.h" + +struct eth_work { + struct work_struct work; + struct mcast_member *member; + struct ib_device *device; + u8 port_num; +}; + +static void deref_member(struct mcast_member *member) +{ + if (atomic_dec_and_test(&member->refcount)) + complete(&member->comp); +} + +static void eth_mcast_work_handler(struct work_struct *work) +{ + struct eth_work *w = container_of(work, struct eth_work, work); + int err; + struct ib_port_attr port_attr; + int status = 0; + + err = ib_query_port(w->device, w->port_num, &port_attr); + if (err) + status = err; + else if (port_attr.state != IB_PORT_ACTIVE) + status = -EAGAIN; + + w->member->multicast.rec.qkey = cpu_to_be32(0xc2c); + err = w->member->multicast.callback(status, &w->member->multicast); + deref_member(w->member); + if (err) + ib_sa_free_multicast(&w->member->multicast); + + kfree(w); +} + +struct ib_sa_multicast * +rdmaoe_sa_join_multicast(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_mcmember_rec *rec, + ib_sa_comp_mask comp_mask, gfp_t gfp_mask, + int (*callback)(int status, + struct ib_sa_multicast *multicast), + void *context) +{ + struct eth_work *w; + struct mcast_member *member; + int err; + + member = kzalloc(sizeof *member, gfp_mask); + if (!member) + return ERR_PTR(-ENOMEM); + + w = kzalloc(sizeof *w, gfp_mask); + if (!w) { + err = -ENOMEM; + goto out1; + } + w->member = member; + w->device = device; + w->port_num = port_num; + + member->multicast.context = context; + member->multicast.callback = callback; + member->client = client; + member->multicast.rec = *rec; + init_completion(&member->comp); + atomic_set(&member->refcount, 1); + + atomic_inc(&member->refcount); + ib_sa_client_get(client); + INIT_WORK(&w->work, eth_mcast_work_handler); + schedule_work(&w->work); + + return &member->multicast; + +out1: + kfree(member); + return ERR_PTR(err); +} +EXPORT_SYMBOL(rdmaoe_sa_join_multicast); + +void rdmaoe_sa_free_multicast(struct ib_sa_multicast *multicast) +{ + struct mcast_member *member; + + member = container_of(multicast, struct mcast_member, multicast); + + deref_member(member); + wait_for_completion(&member->comp); + ib_sa_client_put(member->client); + kfree(member); +} +EXPORT_SYMBOL(rdmaoe_sa_free_multicast); + +struct eth_work_container { + struct work_struct work; + struct ib_sa_path_query *query; +}; + +static void resolve_callback(struct work_struct *work) +{ + struct eth_work_container *eth = + container_of(work, struct eth_work_container, work); + struct ib_sa_path_query *query = eth->query; + struct ib_sa_path_rec res = {}; + + res.dgid = query->dgid; + res.sgid = query->sgid; + res.hop_limit = 2; /* TBD fix this */ + res.mtu = IB_MTU_1024; /* TBD fix me */ + query->callback(0, &res, query->context); + + ib_sa_client_put(query->sa_query.client); +} + +int rdmaoe_sa_path_rec_get(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_path_rec *rec, + ib_sa_comp_mask comp_mask, + int timeout_ms, gfp_t gfp_mask, + void (*callback)(int status, + struct ib_sa_path_rec *resp, + void *context), + void *context, + struct ib_sa_query **sa_query) +{ + struct ib_sa_path_query *query; + struct eth_work_container *eth; + + query = kzalloc(sizeof *query, gfp_mask); + if (!query) + return -ENOMEM; + + eth = kzalloc(sizeof *eth, gfp_mask); + if (!eth) { + kfree(query); + return -ENOMEM; + } + + ib_sa_client_get(client); + query->sa_query.client = client; + query->callback = callback; + query->context = context; + query->dgid = rec->dgid; + query->sgid = rec->sgid; + + *sa_query = &query->sa_query; + + eth->query = query; + INIT_WORK(ð->work, resolve_callback); + schedule_work(ð->work); + + return 0; +} +EXPORT_SYMBOL(rdmaoe_sa_path_rec_get); + diff --git a/drivers/infiniband/core/sa.h b/drivers/infiniband/core/sa.h index b1d4bbf..da780cf 100644 --- a/drivers/infiniband/core/sa.h +++ b/drivers/infiniband/core/sa.h @@ -37,6 +37,30 @@ #include <rdma/ib_sa.h> +struct ib_sa_query { + void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *); + void (*release)(struct ib_sa_query *); + struct ib_sa_client *client; + struct ib_sa_port *port; + struct ib_mad_send_buf *mad_buf; + struct ib_sa_sm_ah *sm_ah; + int id; +}; + +struct ib_sa_service_query { + void (*callback)(int, struct ib_sa_service_rec *, void *); + void *context; + struct ib_sa_query sa_query; +}; + +struct ib_sa_path_query { + void (*callback)(int, struct ib_sa_path_rec *, void *); + void *context; + struct ib_sa_query sa_query; + union ib_gid dgid; + union ib_gid sgid; +}; + static inline void ib_sa_client_get(struct ib_sa_client *client) { atomic_inc(&client->users); diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 1865049..0625e10 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -46,6 +46,7 @@ #include <rdma/ib_pack.h> #include <rdma/ib_cache.h> #include "sa.h" +#include "multicast.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand subnet administration query support"); @@ -72,28 +73,6 @@ struct ib_sa_device { struct ib_sa_port port[0]; }; -struct ib_sa_query { - void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *); - void (*release)(struct ib_sa_query *); - struct ib_sa_client *client; - struct ib_sa_port *port; - struct ib_mad_send_buf *mad_buf; - struct ib_sa_sm_ah *sm_ah; - int id; -}; - -struct ib_sa_service_query { - void (*callback)(int, struct ib_sa_service_rec *, void *); - void *context; - struct ib_sa_query sa_query; -}; - -struct ib_sa_path_query { - void (*callback)(int, struct ib_sa_path_rec *, void *); - void *context; - struct ib_sa_query sa_query; -}; - struct ib_sa_mcmember_query { void (*callback)(int, struct ib_sa_mcmember_rec *, void *); void *context; @@ -363,6 +342,9 @@ static void update_sm_ah(struct work_struct *work) struct ib_port_attr port_attr; struct ib_ah_attr ah_attr; + if (ib_get_port_link_type(port->agent->device, port->port_num) != PORT_LINK_IB) + return; + if (ib_query_port(port->agent->device, port->port_num, &port_attr)) { printk(KERN_WARNING "Couldn't query port\n"); return; diff --git a/include/rdma/rdmaoe_sa.h b/include/rdma/rdmaoe_sa.h new file mode 100644 index 0000000..a672cf7 --- /dev/null +++ b/include/rdma/rdmaoe_sa.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2009 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef RDMAOE_SA_H +#define RDMAOE_SA_H + +#include <rdma/ib_sa.h> + +struct ib_sa_multicast * +rdmaoe_sa_join_multicast(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_mcmember_rec *rec, + ib_sa_comp_mask comp_mask, gfp_t gfp_mask, + int (*callback)(int status, + struct ib_sa_multicast *multicast), + void *context); + +void rdmaoe_sa_free_multicast(struct ib_sa_multicast *multicast); + +int rdmaoe_sa_path_rec_get(struct ib_sa_client *client, + struct ib_device *device, u8 port_num, + struct ib_sa_path_rec *rec, + ib_sa_comp_mask comp_mask, + int timeout_ms, gfp_t gfp_mask, + void (*callback)(int status, + struct ib_sa_path_rec *resp, + void *context), + void *context, + struct ib_sa_query **sa_query); + +#endif /* RDMAOE_SA_H */ + -- 1.6.3.3 ----- End forwarded message ----- _______________________________________________ ewg mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
