Optimized SLtoVLMappingTable programming reduces the number of MADs needed from O(n**2) to O(n). See IBA 1.2.1 vol 1 p. 843 14.2.5.8 SLtoVLMappingTable.
Signed-off-by: Hal Rosenstock <[email protected]> --- Changes since v2: Use osm_log rather than OSM_LOG in osm_slvl_rcv_process for optimized sl2vl mapping programming verbose log message Removed unneeded check of p_physp in input port loop in osm_slvl_rcv_process for optimized sl2vl mapping programming Changes since v1: Changed optimized programming flow in osm_qos to only call sl2vl_update once per switch supporting this feature diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h index 3c08689..583c070 100644 --- a/opensm/include/opensm/osm_subnet.h +++ b/opensm/include/opensm/osm_subnet.h @@ -206,6 +206,7 @@ typedef struct osm_subn_opt { boolean_t daemon; boolean_t sm_inactive; boolean_t babbling_port_policy; + boolean_t use_optimized_slvl; osm_qos_options_t qos_options; osm_qos_options_t qos_ca_options; osm_qos_options_t qos_sw0_options; @@ -433,6 +434,10 @@ typedef struct osm_subn_opt { * babbling_port_policy * OpenSM will enforce its "babbling" port policy. * +* use_optimized_slvl +* Use optimized SLtoVLMappingTable programming if +* device indicates it supports this. +* * perfmgr * Enable or disable the performance manager * diff --git a/opensm/opensm/osm_qos.c b/opensm/opensm/osm_qos.c index 08f9a60..617a86e 100644 --- a/opensm/opensm/osm_qos.c +++ b/opensm/opensm/osm_qos.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2006-2009 Voltaire, Inc. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -150,7 +151,7 @@ static ib_api_status_t vlarb_update(osm_sm_t * sm, osm_physp_t * p, static ib_api_status_t sl2vl_update_table(osm_sm_t * sm, osm_physp_t * p, uint8_t in_port, uint8_t out_port, - unsigned force_update, + unsigned optimize, unsigned force_update, const ib_slvl_table_t * sl2vl_table) { osm_madw_context_t context; @@ -180,41 +181,47 @@ static ib_api_status_t sl2vl_update_table(osm_sm_t * sm, osm_physp_t * p, context.slvl_context.node_guid = osm_node_get_node_guid(p_node); context.slvl_context.port_guid = osm_physp_get_port_guid(p); context.slvl_context.set_method = TRUE; - attr_mod = in_port << 8 | out_port; + if (optimize) + /* wildcard both input and output ports */ + attr_mod = 0x30000; + else + attr_mod = in_port << 8 | out_port; return osm_req_set(sm, osm_physp_get_dr_path_ptr(p), (uint8_t *) & tbl, sizeof(tbl), IB_MAD_ATTR_SLVL_TABLE, cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context); } -static ib_api_status_t sl2vl_update(osm_sm_t * sm, osm_port_t * p_port, +static ib_api_status_t sl2vl_update(osm_sm_t * sm, osm_physp_t * p0, osm_physp_t * p, uint8_t port_num, - unsigned force_update, + unsigned optimize, unsigned force_update, const struct qos_config *qcfg) { ib_api_status_t status; uint8_t i, num_ports; - osm_physp_t *p_physp; + osm_node_t *p_node; - if (osm_node_get_type(osm_physp_get_node_ptr(p)) == IB_NODE_TYPE_SWITCH) { + p_node = osm_physp_get_node_ptr(p); + if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH) { if (ib_port_info_get_vl_cap(&p->port_info) == 1) { /* Check port 0's capability mask */ - p_physp = p_port->p_physp; - if (! - (p_physp->port_info. - capability_mask & IB_PORT_CAP_HAS_SL_MAP)) + if (!(p0->port_info.capability_mask & IB_PORT_CAP_HAS_SL_MAP)) return IB_SUCCESS; } - num_ports = osm_node_get_num_physp(osm_physp_get_node_ptr(p)); + num_ports = osm_node_get_num_physp(p_node); } else { if (!(p->port_info.capability_mask & IB_PORT_CAP_HAS_SL_MAP)) return IB_SUCCESS; num_ports = 1; } + if (optimize) + return sl2vl_update_table(sm, p, 1, port_num, optimize, + force_update, &qcfg->sl2vl); + for (i = 0; i < num_ports; i++) { - status = sl2vl_update_table(sm, p, i, port_num, force_update, - &qcfg->sl2vl); + status = sl2vl_update_table(sm, p, i, port_num, optimize, + force_update, &qcfg->sl2vl); if (status != IB_SUCCESS) return status; } @@ -222,22 +229,30 @@ static ib_api_status_t sl2vl_update(osm_sm_t * sm, osm_port_t * p_port, return IB_SUCCESS; } -static int qos_physp_setup(osm_log_t * p_log, osm_sm_t * sm, - osm_port_t * p_port, osm_physp_t * p, - uint8_t port_num, unsigned force_update, - const struct qos_config *qcfg) +static int vlarb_physp_setup(osm_sm_t * sm, osm_physp_t * p, uint8_t port_num, + unsigned force_update, + const struct qos_config *qcfg) { - ib_api_status_t status; - /* OpVLs should be ok at this moment - just use it */ /* setup VL high limit on the physp later to be updated by link mgr */ p->vl_high_limit = qcfg->vl_high_limit; /* setup VLArbitration */ - status = vlarb_update(sm, p, port_num, force_update, qcfg); + return vlarb_update(sm, p, port_num, force_update, qcfg); +} + +static int qos_physp_setup(osm_log_t * p_log, osm_sm_t * sm, + osm_physp_t * p0, osm_physp_t * p, + uint8_t port_num, unsigned force_update, + const struct qos_config *qcfg) +{ + ib_api_status_t status; + + /* setup VLArbitration */ + status = vlarb_physp_setup(sm, p, port_num, force_update, qcfg); if (status != IB_SUCCESS) { - OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6202 : " + OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6202: " "failed to update VLArbitration tables " "for port %" PRIx64 " #%d\n", cl_ntoh64(p->port_guid), port_num); @@ -245,9 +260,9 @@ static int qos_physp_setup(osm_log_t * p_log, osm_sm_t * sm, } /* setup SL2VL tables */ - status = sl2vl_update(sm, p_port, p, port_num, force_update, qcfg); + status = sl2vl_update(sm, p0, p, port_num, 0, force_update, qcfg); if (status != IB_SUCCESS) { - OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6203 : " + OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6203: " "failed to update SL2VLMapping tables " "for port %" PRIx64 " #%d\n", cl_ntoh64(p->port_guid), port_num); @@ -267,6 +282,7 @@ int osm_qos_setup(osm_opensm_t * p_osm) uint32_t num_physp; osm_physp_t *p_physp; osm_node_t *p_node; + osm_switch_t *p_sw; unsigned force_update; int ret = 0; uint8_t i; @@ -290,6 +306,44 @@ int osm_qos_setup(osm_opensm_t * p_osm) /* read QoS policy config file */ osm_qos_parse_policy_file(&p_osm->subn); + /* loop on switches that support optimized SL2VL programming first */ + p_tbl = &p_osm->subn.sw_guid_tbl; + p_next = cl_qmap_head(p_tbl); + while (p_next != cl_qmap_end(p_tbl)) { + p_sw = (osm_switch_t *) p_next; + p_next = cl_qmap_next(p_next); + + if (ib_switch_info_get_opt_sl2vlmapping(&p_sw->switch_info) && + p_osm->subn.opt.use_optimized_slvl) { + p_physp = osm_node_get_physp_ptr(p_sw->p_node, 1); + num_physp = osm_node_get_num_physp(p_sw->p_node); + force_update = p_osm->subn.need_update; + for (i = 1; i < num_physp; i++) { + p_physp = osm_node_get_physp_ptr(p_sw->p_node, i); + if (!p_physp) + continue; + if (vlarb_physp_setup(&p_osm->sm, p_physp, i, + p_physp->need_update || + p_osm->subn.need_update, + &swe_config)) + ret = -1; + force_update |= p_physp->need_update; + } + if (sl2vl_update(&p_osm->sm, + osm_node_get_physp_ptr(p_sw->p_node, 0), + p_physp, i, 1, force_update, + &swe_config)) { + OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR 6204: " + "failed to update optimized SL2VLMapping" + " tables for port %" PRIx64 " #%d\n", + cl_ntoh64(p_physp->port_guid), i); + ret = -1; + } + } + } + + /* now, loop on ports skipping the external ports of switches + that support optimized SL2VL programming */ p_tbl = &p_osm->subn.port_guid_tbl; p_next = cl_qmap_head(p_tbl); while (p_next != cl_qmap_end(p_tbl)) { @@ -298,6 +352,11 @@ int osm_qos_setup(osm_opensm_t * p_osm) p_node = p_port->p_node; if (p_node->sw) { + /* skip switches with optimized SL2VL mapping + programming since already done */ + if (ib_switch_info_get_opt_sl2vlmapping(&p_node->sw->switch_info) && + p_osm->subn.opt.use_optimized_slvl) + goto check_port0; num_physp = osm_node_get_num_physp(p_node); for (i = 1; i < num_physp; i++) { p_physp = osm_node_get_physp_ptr(p_node, i); @@ -306,10 +365,11 @@ int osm_qos_setup(osm_opensm_t * p_osm) force_update = p_physp->need_update || p_osm->subn.need_update; if (qos_physp_setup(&p_osm->log, &p_osm->sm, - p_port, p_physp, i, + p_port->p_physp, p_physp, i, force_update, &swe_config)) ret = -1; } +check_port0: /* skip base port 0 */ if (!ib_switch_info_is_enhanced_port0 (&p_node->sw->switch_info)) @@ -326,8 +386,8 @@ int osm_qos_setup(osm_opensm_t * p_osm) continue; force_update = p_physp->need_update || p_osm->subn.need_update; - if (qos_physp_setup(&p_osm->log, &p_osm->sm, p_port, p_physp, - 0, force_update, cfg)) + if (qos_physp_setup(&p_osm->log, &p_osm->sm, p_port->p_physp, + p_physp, 0, force_update, cfg)) ret = -1; } diff --git a/opensm/opensm/osm_slvl_map_rcv.c b/opensm/opensm/osm_slvl_map_rcv.c index 4f75690..214a763 100644 --- a/opensm/opensm/osm_slvl_map_rcv.c +++ b/opensm/opensm/osm_slvl_map_rcv.c @@ -2,6 +2,7 @@ * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -70,7 +71,9 @@ void osm_slvl_rcv_process(IN void *context, IN void *p_data) osm_slvl_context_t *p_context; ib_net64_t port_guid; ib_net64_t node_guid; - uint8_t out_port_num, in_port_num; + uint32_t attr_mod; + uint8_t out_port_num, in_port_num, startinport, startoutport, + endinport, endoutport; CL_ASSERT(sm); @@ -109,6 +112,9 @@ void osm_slvl_rcv_process(IN void *context, IN void *p_data) (uint8_t) cl_ntoh32(p_smp->attr_mod & 0xFF000000); in_port_num = (uint8_t) cl_ntoh32((p_smp->attr_mod & 0x00FF0000) << 8); + attr_mod = cl_ntoh32(p_smp->attr_mod); + if (attr_mod & 0x30000) + goto opt_sl2vl; p_physp = osm_node_get_physp_ptr(p_node, out_port_num); } else { p_physp = p_port->p_physp; @@ -121,7 +127,7 @@ void osm_slvl_rcv_process(IN void *context, IN void *p_data) all we want is to update the subnet. */ OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, - "Got SLtoVL get response in_port_num %u out_port_num %u with " + "Received SLtoVL GetResp in_port_num %u out_port_num %u with " "GUID 0x%" PRIx64 " for parent node GUID 0x%" PRIx64 ", TID 0x%" PRIx64 "\n", in_port_num, out_port_num, cl_ntoh64(port_guid), cl_ntoh64(node_guid), cl_ntoh64(p_smp->trans_id)); @@ -140,6 +146,38 @@ void osm_slvl_rcv_process(IN void *context, IN void *p_data) out_port_num, p_slvl_tbl, OSM_LOG_DEBUG); osm_physp_set_slvl_tbl(p_physp, p_slvl_tbl, in_port_num); + goto Exit; + +opt_sl2vl: + if (osm_log_is_active(sm->p_log, OSM_LOG_VERBOSE)) + osm_log(sm->p_log, OSM_LOG_VERBOSE, + "Received optimized SLtoVL get response in_port_num %u " + "out_port_num %u with GUID 0x%" PRIx64 " for parent " + "node GUID 0x%" PRIx64 ", TID 0x%" PRIx64 "\n", + in_port_num, out_port_num, cl_ntoh64(port_guid), + cl_ntoh64(node_guid), cl_ntoh64(p_smp->trans_id)); + + osm_dump_slvl_map_table(sm->p_log, port_guid, in_port_num, + out_port_num, p_slvl_tbl, OSM_LOG_DEBUG); + + if (attr_mod & 0x10000) { + startoutport = ib_switch_info_is_enhanced_port0(&p_node->sw->switch_info) ? 0 : 1; + endoutport = osm_node_get_num_physp(p_node); + } else + endoutport = startoutport = out_port_num; + if (attr_mod & 0x20000) { + startinport = ib_switch_info_is_enhanced_port0(&p_node->sw->switch_info) ? 0 : 1; + endinport = osm_node_get_num_physp(p_node); + } else + endinport = startinport = in_port_num; + + for (out_port_num = startoutport; out_port_num < endoutport; + out_port_num++) { + p_physp = osm_node_get_physp_ptr(p_node, out_port_num); + for (in_port_num = startinport; in_port_num < endinport; + in_port_num++) + osm_physp_set_slvl_tbl(p_physp, p_slvl_tbl, in_port_num); + } Exit: cl_plock_release(sm->p_lock); diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c index 09029db..103b5c6 100644 --- a/opensm/opensm/osm_subnet.c +++ b/opensm/opensm/osm_subnet.c @@ -354,6 +354,7 @@ static const opt_rec_t opt_tbl[] = { { "daemon", OPT_OFFSET(daemon), opts_parse_boolean, NULL, 0 }, { "sm_inactive", OPT_OFFSET(sm_inactive), opts_parse_boolean, NULL, 1 }, { "babbling_port_policy", OPT_OFFSET(babbling_port_policy), opts_parse_boolean, NULL, 1 }, + { "use_optimized_slvl", OPT_OFFSET(use_optimized_slvl), opts_parse_boolean, NULL, 1 }, #ifdef ENABLE_OSM_PERF_MGR { "perfmgr", OPT_OFFSET(perfmgr), opts_parse_boolean, NULL, 0 }, { "perfmgr_redir", OPT_OFFSET(perfmgr_redir), opts_parse_boolean, NULL, 0 }, @@ -713,6 +714,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt) p_opt->daemon = FALSE; p_opt->sm_inactive = FALSE; p_opt->babbling_port_policy = FALSE; + p_opt->use_optimized_slvl = FALSE; #ifdef ENABLE_OSM_PERF_MGR p_opt->perfmgr = FALSE; p_opt->perfmgr_redir = TRUE; @@ -1499,10 +1501,13 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts) "# SM Inactive\n" "sm_inactive %s\n\n" "# Babbling Port Policy\n" - "babbling_port_policy %s\n\n", + "babbling_port_policy %s\n\n" + "# Use Optimized SLtoVLMapping programming if supported by device\n" + "use_optimized_slvl %s\n\n", p_opts->daemon ? "TRUE" : "FALSE", p_opts->sm_inactive ? "TRUE" : "FALSE", - p_opts->babbling_port_policy ? "TRUE" : "FALSE"); + p_opts->babbling_port_policy ? "TRUE" : "FALSE", + p_opts->use_optimized_slvl ? "TRUE" : "FALSE"); #ifdef ENABLE_OSM_PERF_MGR fprintf(out, -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
