move lid/gid queries from the connection setup phase and put them in the open call to avoid overhead of more fd's during connections. No need to query during connection setup since uDAPL binds to specific hca/ports via dat_ia_open.
Signed-off-by: Arlin Davis <[email protected]> --- dapl/openib_scm/dapl_ib_cm.c | 60 +++------------------------------------ dapl/openib_scm/dapl_ib_util.c | 32 +++++++++++++++++++++ dapl/openib_scm/dapl_ib_util.h | 4 ++- 3 files changed, 40 insertions(+), 56 deletions(-) diff --git a/dapl/openib_scm/dapl_ib_cm.c b/dapl/openib_scm/dapl_ib_cm.c index 9a15e42..9defc42 100644 --- a/dapl/openib_scm/dapl_ib_cm.c +++ b/dapl/openib_scm/dapl_ib_cm.c @@ -285,16 +285,6 @@ static void dapli_cm_queue(struct ib_cm_handle *cm_ptr) strerror(errno)); } -static uint16_t dapli_get_lid(IN struct ibv_context *ctx, IN uint8_t port) -{ - struct ibv_port_attr port_attr; - - if(ibv_query_port(ctx, port,&port_attr)) - return(0xffff); - else - return(port_attr.lid); -} - /* * ACTIVE/PASSIVE: called from CR thread or consumer via ep_disconnect */ @@ -468,28 +458,9 @@ dapli_socket_connect(DAPL_EP *ep_ptr, cm_ptr->dst.qp_type = htons(ep_ptr->qp_handle->qp_type); #endif cm_ptr->dst.port = htons(ia_ptr->hca_ptr->port_num); - cm_ptr->dst.lid = - htons(dapli_get_lid(ia_ptr->hca_ptr->ib_hca_handle, - (uint8_t)ia_ptr->hca_ptr->port_num)); - if (cm_ptr->dst.lid == 0xffff) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONNECT: query LID ERR %s -> %s\n", - strerror(errno), - inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr)); - goto bail; - } - - /* in network order */ - if (ibv_query_gid(ia_ptr->hca_ptr->ib_hca_handle, - (uint8_t)ia_ptr->hca_ptr->port_num, - 0, &cm_ptr->dst.gid)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " CONNECT: query GID ERR %s -> %s\n", - strerror(errno), - inet_ntoa(((struct sockaddr_in *)r_addr)->sin_addr)); - goto bail; - } - + cm_ptr->dst.lid = ia_ptr->hca_ptr->ib_trans.lid; + cm_ptr->dst.gid = ia_ptr->hca_ptr->ib_trans.gid; + /* save references */ cm_ptr->hca = ia_ptr->hca_ptr; cm_ptr->ep = ep_ptr; @@ -958,29 +929,8 @@ dapli_socket_accept_usr(DAPL_EP *ep_ptr, local.qpn = htonl(ep_ptr->qp_handle->qp_num); local.qp_type = htons(ep_ptr->qp_handle->qp_type); local.port = htons(ia_ptr->hca_ptr->port_num); - local.lid = htons(dapli_get_lid(ia_ptr->hca_ptr->ib_hca_handle, - (uint8_t)ia_ptr->hca_ptr->port_num)); - if (local.lid == 0xffff) { - dapl_log(DAPL_DBG_TYPE_ERR, - " ACCEPT_USR: query LID ERR %s -> %s\n", - strerror(errno), - inet_ntoa(((struct sockaddr_in *) - &cm_ptr->dst.ia_address)->sin_addr)); - goto bail; - } - - /* in network order */ - if (ibv_query_gid(ia_ptr->hca_ptr->ib_hca_handle, - (uint8_t)ia_ptr->hca_ptr->port_num, - 0, &local.gid)) { - dapl_log(DAPL_DBG_TYPE_ERR, - " ACCEPT_USR: query GID ERR %s -> %s\n", - strerror(errno), - inet_ntoa(((struct sockaddr_in *) - &cm_ptr->dst.ia_address)->sin_addr)); - goto bail; - } - + local.lid = ia_ptr->hca_ptr->ib_trans.lid; + local.gid = ia_ptr->hca_ptr->ib_trans.gid; local.ia_address = ia_ptr->hca_ptr->hca_address; local.p_size = htonl(p_size); iov[0].iov_base = (void *) &local; diff --git a/dapl/openib_scm/dapl_ib_util.c b/dapl/openib_scm/dapl_ib_util.c index fdbae3c..391a0e5 100644 --- a/dapl/openib_scm/dapl_ib_util.c +++ b/dapl/openib_scm/dapl_ib_util.c @@ -196,6 +196,7 @@ DAT_RETURN dapls_ib_open_hca ( IN DAPL_HCA *hca_ptr) { struct ibv_device **dev_list; + struct ibv_port_attr port_attr; int i; DAT_RETURN dat_status = DAT_SUCCESS; @@ -238,6 +239,30 @@ found: goto err; } + /* get lid for this hca-port, network order */ + if(ibv_query_port(hca_ptr->ib_hca_handle, + (uint8_t)hca_ptr->port_num, + &port_attr)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: get lid ERR for %s, err=%s\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + strerror(errno)); + goto err; + } else { + hca_ptr->ib_trans.lid = htons(port_attr.lid); + } + + /* get gid for this hca-port, network order */ + if (ibv_query_gid(hca_ptr->ib_hca_handle, + (uint8_t)hca_ptr->port_num, + 0, &hca_ptr->ib_trans.gid)) { + dapl_log(DAPL_DBG_TYPE_ERR, + " open_hca: query GID ERR for %s, err=%s\n", + ibv_get_device_name(hca_ptr->ib_trans.ib_dev), + strerror(errno)); + goto err; + } + /* set RC tunables via enviroment or default */ hca_ptr->ib_trans.max_inline_send = dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT); @@ -325,6 +350,13 @@ found: hca_ptr->port_num, inet_ntoa(((struct sockaddr_in *) &hca_ptr->hca_address)->sin_addr)); + dapl_dbg_log(DAPL_DBG_TYPE_UTIL, + " open_hca: LID 0x%x GID Subnet 0x"F64x" ID 0x"F64x"\n", + ntohs(hca_ptr->ib_trans.lid), + (unsigned long long) + htonll(hca_ptr->ib_trans.gid.global.subnet_prefix), + (unsigned long long) + htonll(hca_ptr->ib_trans.gid.global.interface_id)); ibv_free_device_list(dev_list); return dat_status; diff --git a/dapl/openib_scm/dapl_ib_util.h b/dapl/openib_scm/dapl_ib_util.h index fd1c24e..7c3251a 100644 --- a/dapl/openib_scm/dapl_ib_util.h +++ b/dapl/openib_scm/dapl_ib_util.h @@ -277,12 +277,13 @@ typedef enum /* ib_hca_transport_t, specific to this implementation */ typedef struct _ib_hca_transport { + union ibv_gid gid; struct ibv_device *ib_dev; ib_cq_handle_t ib_cq_empty; DAPL_OS_LOCK cq_lock; int max_inline_send; ib_thread_state_t cq_state; - DAPL_OS_THREAD cq_thread; + DAPL_OS_THREAD cq_thread; struct ibv_comp_channel *ib_cq; int cr_state; DAPL_OS_THREAD thread; @@ -293,6 +294,7 @@ typedef struct _ib_hca_transport ib_async_cq_handler_t async_cq_error; ib_async_dto_handler_t async_cq; ib_async_qp_handler_t async_qp_error; + uint16_t lid; uint8_t ack_timer; uint8_t ack_retry; uint8_t rnr_timer; -- 1.5.2.5 _______________________________________________ general mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
