Hi Sasha,
I agree with most of your comments.
See below:
Sasha Khapyorsky wrote:
Hi Yevgeny,
The initial comments below.
Basically I think some code cleanup is needed, and please decrease
number of osm_log(...OSM_LOG_DEBUG...).
Sasha
On 15:15 Mon 03 Sep , Yevgeny Kliteynik wrote:
Selecting path according to QoS policy level that
the PathRecord query matches.
Signed-off-by: Yevgeny Kliteynik <[EMAIL PROTECTED]>
---
opensm/opensm/osm_sa_path_record.c | 383 ++++++++++++++++++++++++++++++------
1 files changed, 320 insertions(+), 63 deletions(-)
diff --git a/opensm/opensm/osm_sa_path_record.c
b/opensm/opensm/osm_sa_path_record.c
index 1b781f0..8fc5eac 100644
--- a/opensm/opensm/osm_sa_path_record.c
+++ b/opensm/opensm/osm_sa_path_record.c
@@ -67,6 +67,7 @@
#include <opensm/osm_multicast.h>
#include <opensm/osm_partition.h>
#include <opensm/osm_opensm.h>
+#include <opensm/osm_qos_policy.h>
#ifdef ROUTER_EXP
#include <opensm/osm_router.h>
#include <opensm/osm_sa_mcmember_record.h>
@@ -236,8 +237,9 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
{
const osm_node_t *p_node;
const osm_physp_t *p_physp;
+ const osm_physp_t *p_src_physp;
const osm_physp_t *p_dest_physp;
- const osm_prtn_t *p_prtn;
+ const osm_prtn_t *p_prtn = NULL;
const ib_port_info_t *p_pi;
ib_api_status_t status = IB_SUCCESS;
ib_net16_t pkey;
@@ -248,14 +250,24 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
uint8_t required_rate;
uint8_t required_pkt_life;
uint8_t sl;
+ uint8_t in_port_num;
ib_net16_t dest_lid;
+ uint8_t i;
+ uint8_t vl;
+ ib_slvl_table_t *p_slvl_tbl = NULL;
+ boolean_t valid_sls[IB_MAX_NUM_VLS];
Use here uint16_t sl_mask instead of array - flow will be simpler.
No, it won't.
It will save three lines in the end when checking whether there is
a valid sl that doesn't lead to VL15, but it will compilcate a bit
rest of the related code, because I still need to read port's SL2VL
table values one by one and mark them in the array (or bitmap) one
by one.
+ boolean_t sl2vl_valid_path;
+ uint8_t first_valid_sl;
+ osm_qos_level_t *p_qos_level = NULL;
OSM_LOG_ENTER(p_rcv->p_log, __osm_pr_rcv_get_path_parms);
+ memset(valid_sls, TRUE, sizeof(valid_sls));
dest_lid = cl_hton16(dest_lid_ho);
p_dest_physp = p_dest_port->p_physp;
p_physp = p_src_port->p_physp;
+ p_src_physp = p_physp;
p_pi = &p_physp->port_info;
mtu = ib_port_info_get_mtu_cap(p_pi);
@@ -288,13 +300,16 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
p_node = osm_physp_get_node_ptr(p_physp);
if (p_node->sw) {
+ /* source node is a switch */
+ in_port_num = osm_physp_get_port_num(p_physp);
+
/*
* If the dest_lid_ho is equal to the lid of the switch pointed
by
* p_sw then p_physp will be the physical port of the switch
port zero.
+ * Make sure that p_physp points to the out port of the
+ * switch that routes to the destination lid (dest_lid_ho)
*/
- p_physp =
- osm_switch_get_route_by_lid(p_node->sw,
- cl_ntoh16(dest_lid_ho));
+ p_physp = osm_switch_get_route_by_lid(p_node->sw, dest_lid);
if (p_physp == 0) {
osm_log(p_rcv->p_log, OSM_LOG_ERROR,
"__osm_pr_rcv_get_path_parms: ERR 1F02: "
@@ -304,17 +319,36 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
status = IB_NOT_FOUND;
goto Exit;
}
+ if (!p_rcv->p_subn->opt.no_qos)
+ p_slvl_tbl = osm_physp_get_slvl_tbl(p_physp,
in_port_num);
Here
+ }
+
+ if (!p_rcv->p_subn->opt.no_qos) {
+ if (p_node->sw)
+ p_slvl_tbl = osm_physp_get_slvl_tbl(p_physp,
in_port_num);
+ else
+ p_slvl_tbl = osm_physp_get_slvl_tbl(p_physp, 0);
and here - is it double initialization?
Fixed
+
+ /* update valid SLs that still exist on this route */
+ for (i = 0; i < IB_MAX_NUM_VLS; i++) {
+ if (valid_sls[i]) {
+ vl = ib_slvl_table_get(p_slvl_tbl, i);
+ if (vl == IB_DROP_VL)
+ valid_sls[i] = FALSE;
+ }
+ }
}
/*
- * Same as above
+ * now get pointer to the destination port (same as above)
What was wrong with comment? Is not 'p_dest_physp = ' clear?
Fixed
*/
p_node = osm_physp_get_node_ptr(p_dest_physp);
if (p_node->sw) {
- p_dest_physp =
- osm_switch_get_route_by_lid(p_node->sw,
- cl_ntoh16(dest_lid_ho));
+ /*
+ * if destination is switch, we want p_dest_physp to point to
port 0
+ */
+ p_dest_physp = osm_switch_get_route_by_lid(p_node->sw,
dest_lid);
if (p_dest_physp == 0) {
osm_log(p_rcv->p_log, OSM_LOG_ERROR,
@@ -328,6 +362,10 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
}
+ /*
+ * Now go through the path step by step
+ */
+
while (p_physp != p_dest_physp) {
p_physp = osm_physp_get_remote(p_physp);
@@ -341,6 +379,8 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
goto Exit;
}
+ in_port_num = osm_physp_get_port_num(p_physp);
+
/*
This is point to point case (no switch in between)
*/
@@ -409,6 +449,20 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
CL_ASSERT(p_physp);
CL_ASSERT(osm_physp_is_valid(p_physp));
+ p_node = osm_physp_get_node_ptr(p_physp);
+ if (!p_node->sw) {
+ /*
+ * There is some sort of problem in the subnet object!
+ * If this isn't a switch, we should have reached
+ * the destination by now!
+ */
+ osm_log(p_rcv->p_log, OSM_LOG_ERROR,
+ "__osm_pr_rcv_get_path_parms: ERR 1F05: "
+ "Internal error, bad path\n");
+ status = IB_ERROR;
+ goto Exit;
+ }
+
p_pi = &p_physp->port_info;
if (mtu > ib_port_info_get_mtu_cap(p_pi)) {
@@ -435,6 +489,21 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
osm_physp_get_port_num(p_physp));
}
+ if (!p_rcv->p_subn->opt.no_qos) {
+ /*
+ * Check SL2VL table of the switch and update valid SLs
+ */
+ p_slvl_tbl = osm_physp_get_slvl_tbl(p_physp,
in_port_num);
+ for (i = 0; i < IB_MAX_NUM_VLS; i++) {
+ if (valid_sls[i]) {
+ vl = ib_slvl_table_get(p_slvl_tbl, i);
+ if (vl == IB_DROP_VL)
+ valid_sls[i] = FALSE;
+ }
+ }
+ }
+
+ /* go to the next step in the path */
Please drop this useless comment.
Fixed
}
/*
@@ -467,9 +536,118 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
"__osm_pr_rcv_get_path_parms: "
"Path min MTU = %u, min rate = %u\n", mtu, rate);
+ if (!p_rcv->p_subn->opt.no_qos) {
+ /* check whether there is some SL that won't lead to VL15
eventually */
+ sl2vl_valid_path = FALSE;
+ for (i = 0; i < IB_MAX_NUM_VLS; i++) {
+ if (valid_sls[i]) {
+ sl2vl_valid_path = TRUE;
+ first_valid_sl = i;
+ break;
+ }
+ }
+
+ if (!sl2vl_valid_path) {
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "All the SLs lead to VL15 on this
path\n");
+ }
+ status = IB_NOT_FOUND;
+ goto Exit;
+ }
+ }
+
+ if (!p_rcv->p_subn->opt.no_qos && p_rcv->p_subn->p_qos_policy) {
+ /* Get QoS Level object according to the path request */
+ osm_qos_policy_get_qos_level_by_pr(p_rcv->p_subn->p_qos_policy,
+ p_rcv, p_pr,
+ p_src_physp, p_dest_physp,
+ comp_mask, &p_qos_level);
+
+ if (p_qos_level
+ && osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "PathRecord request matches QoS Level '%s'
(%s)\n",
+ p_qos_level->name,
+ (p_qos_level->use) ? p_qos_level->
+ use : "no description");
+ }
+ }
+
+ /* Adjust path parameters according to QoS settings */
+
+ if (p_qos_level) {
+ /* adjust MTU limit according to QoS constraints */
+ if (p_qos_level->mtu_limit_set
+ && (mtu > p_qos_level->mtu_limit)) {
+ mtu = p_qos_level->mtu_limit;
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "QoS constaraints: new smallest MTU =
%u\n",
+ mtu);
+ }
+ }
+
+ /* adjust Rate limit according to QoS constraints */
+ if (p_qos_level->rate_limit_set
+ && (rate > p_qos_level->rate_limit)) {
+ rate = p_qos_level->rate_limit;
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "QoS constaraints: new smallest Rate =
%u\n",
+ rate);
+ }
+ }
+
+ /* adjust Packet Lifetime according to QoS constraints */
+ if (p_qos_level->pkt_life_set
+ && (pkt_life > p_qos_level->pkt_life)) {
+ pkt_life = p_qos_level->pkt_life;
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "QoS constaraints: new smallest Packet
Lifetime = %u\n",
+ pkt_life);
+ }
+ }
+
+ /* adjust SL according to QoS constraints */
+ if (p_qos_level->sl_set) {
+ if (!valid_sls[p_qos_level->sl]) {
+ status = IB_NOT_FOUND;
+ goto Exit;
+ }
+ sl = p_qos_level->sl;
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "QoS constaraints: new SL = %u\n",
+ sl);
+ }
+ }
Please drop all osm_log(..OSM_LOG_DEBUG..) in this block - not each
single line should be logged. If you think that those parameters may be
useful for debugging put final values in single osm_log() somewhere at
end of PR generator.
OK
+ }
+
+ /*
+ * Set packet lifetime.
+ * According to spec definition IBA 1.2 Table 205
+ * PacketLifeTime description, for loopback paths,
+ * packetLifeTime shall be zero.
+ */
+ if (p_src_port == p_dest_port)
+ pkt_life = 0;
+ else
+ if ( !(p_qos_level && p_qos_level->pkt_life_set) )
+ pkt_life = OSM_DEFAULT_SUBNET_TIMEOUT;
+
+
/*
- Determine if these values meet the user criteria
- and adjust appropriately
+ * Done adjusting parameters according to QoS constraints.
+ * Determine if these values meet the user criteria and
+ * adjust appropriately.
*/
/* we silently ignore cases where only the MTU selector is defined */
@@ -511,6 +689,8 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
break;
}
}
+ if (status != IB_SUCCESS)
+ goto Exit;
/* we silently ignore cases where only the Rate selector is defined */
if ((comp_mask & IB_PR_COMPMASK_RATESELEC) &&
@@ -551,14 +731,8 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
break;
}
}
-
- /* Verify the pkt_life_time */
- /* According to spec definition IBA 1.2 Table 205 PacketLifeTime
description,
- for loopback paths, packetLifeTime shall be zero. */
- if (p_src_port == p_dest_port)
- pkt_life = 0; /* loopback */
- else
- pkt_life = OSM_DEFAULT_SUBNET_TIMEOUT;
+ if (status != IB_SUCCESS)
+ goto Exit;
/* we silently ignore cases where only the PktLife selector is defined
*/
if ((comp_mask & IB_PR_COMPMASK_PKTLIFETIMESELEC) &&
@@ -603,38 +777,68 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
if (status != IB_SUCCESS)
goto Exit;
- if (comp_mask & IB_PR_COMPMASK_RAWTRAFFIC &&
- cl_ntoh32(p_pr->hop_flow_raw) & (1 << 31))
- pkey = osm_physp_find_common_pkey(p_physp, p_dest_physp);
- else if (comp_mask & IB_PR_COMPMASK_PKEY) {
- pkey = p_pr->pkey;
- if (!osm_physp_share_this_pkey(p_physp, p_dest_physp, pkey)) {
- osm_log(p_rcv->p_log, OSM_LOG_ERROR,
- "__osm_pr_rcv_get_path_parms: ERR 1F1A: "
- "Ports do not share specified PKey 0x%04x\n",
- cl_ntoh16(pkey));
- status = IB_NOT_FOUND;
- goto Exit;
- }
- } else {
- pkey = osm_physp_find_common_pkey(p_physp, p_dest_physp);
- if (!pkey) {
- osm_log(p_rcv->p_log, OSM_LOG_ERROR,
- "__osm_pr_rcv_get_path_parms: ERR 1F1B: "
- "Ports do not have any shared PKeys\n");
- status = IB_NOT_FOUND;
- goto Exit;
+ /*
+ * set Pkey for this path record request
+ */
+
+ if ((comp_mask & IB_PR_COMPMASK_RAWTRAFFIC) &&
+ (cl_ntoh32(p_pr->hop_flow_raw) & (1 << 31)))
No extra () was needed - this generates confused diff lines.
No sure what you mean here by "confused diff lines".
I agree that the extra () are not *needed*, but isn't
if ((comp_mask & IB_PR_COMPMASK_RAWTRAFFIC) &&
(cl_ntoh32(p_pr->hop_flow_raw) & (1 << 31)))
is more readable than
if (comp_mask & IB_PR_COMPMASK_RAWTRAFFIC &&
cl_ntoh32(p_pr->hop_flow_raw) & 1 << 31)
?
+ pkey = osm_physp_find_common_pkey(p_src_physp, p_dest_physp);
+ else {
+ if (comp_mask & IB_PR_COMPMASK_PKEY) {
+ /*
+ * PR request has a specific pkey:
+ * Check that source and destination share this pkey.
+ * If QoS level has pkeys, check that this pkey exists
+ * in the QoS level pkeys.
+ * PR returned pkey is the requested pkey.
+ */
+ pkey = p_pr->pkey;
+ if (!osm_physp_share_this_pkey(p_src_physp,
p_dest_physp, pkey)) {
+ osm_log(p_rcv->p_log, OSM_LOG_ERROR,
+ "__osm_pr_rcv_get_path_parms: ERR 1F1A:
"
+ "Ports do not share specified PKey
0x%04x\n",
+ cl_ntoh16(pkey));
+ status = IB_NOT_FOUND;
+ goto Exit;
+ }
+ if (p_qos_level && p_qos_level->pkey_range_len &&
+ !osm_qos_level_has_pkey(p_qos_level, pkey)) {
+ osm_log(p_rcv->p_log, OSM_LOG_ERROR,
+ "__osm_pr_rcv_get_path_parms: ERR 1F1D:
"
+ "Ports do not share PKeys defined by QoS
level\n");
+ status = IB_NOT_FOUND;
+ goto Exit;
+ }
+ } else {
+ /* PR request doesn't have a specific pkey */
+
+ if (p_qos_level && p_qos_level->pkey_range_len) {
+ /* If QoS level has pkeys, get shared pkey from
QoS level pkeys */
+ pkey =
osm_qos_level_get_shared_pkey(p_qos_level,
+
p_src_physp,
+
p_dest_physp);
+ if (!pkey) {
+ osm_log(p_rcv->p_log, OSM_LOG_ERROR,
+ "__osm_pr_rcv_get_path_parms: ERR
1F1E: "
+ "Ports do not share PKeys defined by
QoS level\n");
+ status = IB_NOT_FOUND;
+ goto Exit;
+ }
+ } else {
+ pkey = osm_physp_find_common_pkey(p_src_physp,
+ p_dest_physp);
+ if (!pkey) {
+ osm_log(p_rcv->p_log, OSM_LOG_ERROR,
+ "__osm_pr_rcv_get_path_parms: ERR
1F1B: "
+ "Ports do not have any shared
PKeys\n");
+ status = IB_NOT_FOUND;
+ goto Exit;
+ }
+ }
}
}
Please arrange the code above as:
if ()
...
else if ()
...
else
...
OK
-- Yevgeny
, and please try to not exeed 80 chars in the line.
- if (p_rcv->p_subn->opt.routing_engine_name &&
- strcmp(p_rcv->p_subn->opt.routing_engine_name, "lash") == 0)
- /* slid and dest_lid are stored in network in lash */
- sl = osm_get_lash_sl(p_rcv->p_subn->p_osm, p_src_port,
- p_dest_port);
- else
- sl = OSM_DEFAULT_SL;
-
if (pkey) {
p_prtn =
(osm_prtn_t *) cl_qmap_get(&p_rcv->p_subn->prtn_pkey_tbl,
@@ -642,34 +846,87 @@ __osm_pr_rcv_get_path_parms(IN osm_pr_rcv_t * const p_rcv,
0x8000));
if (p_prtn ==
(osm_prtn_t *) cl_qmap_end(&p_rcv->p_subn->prtn_pkey_tbl))
+ p_prtn = NULL;
+ }
+
+ /*
+ * Set PathRecord SL.
+ *
+ * ToDo: What about QoS and LASH routing? How can they coexist?
+ * And what happens when there's a pkey, hence there is a
+ * partition with a certain SL, and this SL doesn't match
+ * the one that's defined by LASH?
+ */
+
+ if (comp_mask & IB_PR_COMPMASK_SL) {
+ /*
+ * Specific SL was requested
+ */
+ sl = ib_path_rec_sl(p_pr);
+ if (p_qos_level && p_qos_level->sl_set && (p_qos_level->sl !=
sl)) {
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "QoS constaraints: required PR SL (%u)
doesn't match QoS SL (%u)\n",
+ sl, p_qos_level->sl);
+ }
+ status = IB_NOT_FOUND;
+ goto Exit;
+ }
+ } else if (p_qos_level && p_qos_level->sl_set) {
+ /*
+ * No specific SL was requested,
+ * but there is an SL in QoS level
+ */
+ sl = p_qos_level->sl;
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ if (pkey && p_prtn && p_prtn->sl != p_qos_level->sl) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "QoS level SL (%u) overrides partition SL
(%u)\n",
+ p_qos_level->sl, p_prtn->sl);
+ }
+ }
+ } else if (pkey) {
+ /*
+ * No specific SL in request or in QoS level - use partition SL
+ */
+ if (!p_prtn) {
/* this may be possible when pkey tables are created
somehow in
previous runs or things are going wrong here */
osm_log(p_rcv->p_log, OSM_LOG_ERROR,
"__osm_pr_rcv_get_path_parms: ERR 1F1C: "
"No partition found for PKey 0x%04x - using default
SL %d\n",
cl_ntoh16(pkey), sl);
- else {
- if (p_rcv->p_subn->opt.routing_engine_name &&
- strcmp(p_rcv->p_subn->opt.routing_engine_name,
- "lash") == 0)
- /* slid and dest_lid are stored in network in
lash */
- sl = osm_get_lash_sl(p_rcv->p_subn->p_osm,
- p_src_port, p_dest_port);
- else
- sl = p_prtn->sl;
- }
-
- /* reset pkey when raw traffic */
- if (comp_mask & IB_PR_COMPMASK_RAWTRAFFIC &&
- cl_ntoh32(p_pr->hop_flow_raw) & (1 << 31))
- pkey = 0;
+ } else
+ sl = p_prtn->sl;
+ } else if (p_rcv->p_subn->opt.routing_engine_name &&
+ strcmp(p_rcv->p_subn->opt.routing_engine_name, "lash") == 0)
{
+ /* slid and dest_lid are stored in network in lash */
+ sl = osm_get_lash_sl(p_rcv->p_subn->p_osm,
+ p_src_port, p_dest_port);
+ } else if (!p_rcv->p_subn->opt.no_qos) {
+ sl = first_valid_sl;
}
+ else
+ sl = OSM_DEFAULT_SL;
- if ((comp_mask & IB_PR_COMPMASK_SL) && ib_path_rec_sl(p_pr) != sl) {
+ if (!p_rcv->p_subn->opt.no_qos && !valid_sls[sl]) {
+ /* selected SL will eventually lead to VL15 */
+ if (osm_log_is_active(p_rcv->p_log, OSM_LOG_DEBUG)) {
+ osm_log(p_rcv->p_log, OSM_LOG_DEBUG,
+ "__osm_pr_rcv_get_path_parms: "
+ "Selected SL (%u) leads to VL15\n", p_prtn->sl);
+ }
status = IB_NOT_FOUND;
goto Exit;
}
+ /* reset pkey when raw traffic */
+ if (comp_mask & IB_PR_COMPMASK_RAWTRAFFIC &&
+ cl_ntoh32(p_pr->hop_flow_raw) & (1 << 31))
+ pkey = 0;
+
p_parms->mtu = mtu;
p_parms->rate = rate;
p_parms->pkt_life = pkt_life;
--
1.5.1.4
_______________________________________________
general mailing list
[email protected]
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
