On Thu, 24 May 2012 15:10:23 +0300 Alex Netes <[email protected]> wrote:
> The issue is resolved by adding High priority unicast FIFO queue to the VL15 > poller thread. > > Signed-off-by: Vladimir Koushnir <[email protected]> > Signed-off-by: Alex Netes <[email protected]> > --- > include/opensm/osm_sm.h | 7 +++++-- > include/opensm/osm_vl15intf.h | 15 +++++++++++++-- > opensm/osm_req.c | 6 +++--- > opensm/osm_resp.c | 5 +++-- > opensm/osm_sm_mad_ctrl.c | 2 +- > opensm/osm_sminfo_rcv.c | 8 ++++---- > opensm/osm_trap_rcv.c | 2 +- > opensm/osm_vl15intf.c | 36 +++++++++++++++++++++++++++++------- > 8 files changed, 59 insertions(+), 22 deletions(-) > > diff --git a/include/opensm/osm_sm.h b/include/opensm/osm_sm.h > index 4c7b120..17d6afd 100644 > --- a/include/opensm/osm_sm.h > +++ b/include/opensm/osm_sm.h > @@ -495,7 +495,8 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const > osm_dr_path_t * p_path, > ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > IN const osm_madw_t * p_req_madw, > IN ib_net16_t status, > - IN const uint8_t * p_payload); > + IN const uint8_t * p_payload, > + IN boolean_t force_flag); > /* > * PARAMETERS > * p_resp > @@ -510,7 +511,9 @@ ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > * > * p_payload > * [in] Pointer to the payload of the response MAD. > -* > +* force_flag > +* [in] If TRUE, indicates that the response should be processed > immediately > +* by VL15 poller thread Shouldn't "force_flag" be something like "immediate"? Same comment for all callers of this function. > * RETURN VALUES > * IB_SUCCESS if the response was successful. > * > diff --git a/include/opensm/osm_vl15intf.h b/include/opensm/osm_vl15intf.h > index e621c68..a600de6 100644 > --- a/include/opensm/osm_vl15intf.h > +++ b/include/opensm/osm_vl15intf.h > @@ -123,6 +123,7 @@ typedef struct osm_vl15 { > cl_thread_t poller; > cl_qlist_t rfifo; > cl_qlist_t ufifo; > + cl_qlist_t hp_ufifo; > cl_spinlock_t lock; > osm_vendor_t *p_vend; > osm_log_t *p_log; > @@ -156,6 +157,10 @@ typedef struct osm_vl15 { > * First-in First-out queue for outbound VL15 MADs for which > * no response is expected, aka the "unicast fifo". > * > +* hp_ufifo > +* First-in First-out high priority queue for outbound VL15 MADs > for which > +* no response is expected. SMInfo response MADs are placed in > this queue > +* > * poller > * Worker thread pool that services the fifo to transmit VL15 MADs > * > @@ -298,7 +303,7 @@ ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl15, IN > osm_vendor_t * p_vend, > * > * SYNOPSIS > */ > -void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw); > +void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw, IN > boolean_t force_flag); > /* > * PARAMETERS > * p_vl15 > @@ -307,6 +312,9 @@ void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t > * p_madw); > * p_madw > * [in] Pointer to a MAD wrapper structure containing the MAD. > * > +* force_flag > +* [in] If set to TRUE, high priority ucast queue should be used > +* or "high_priority"? Same comment for all callers of this function. Ira > * RETURN VALUES > * This function does not return a value. > * > @@ -327,12 +335,15 @@ void osm_vl15_post(IN osm_vl15_t * p_vl15, IN > osm_madw_t * p_madw); > * > * SYNOPSIS > */ > -void osm_vl15_poll(IN osm_vl15_t * p_vl); > +void osm_vl15_poll(IN osm_vl15_t * p_vl, IN boolean_t force_flag); > /* > * PARAMETERS > * p_vl15 > * [in] Pointer to an osm_vl15_t object. > * > +* force_flag > +* [in] If set to TRUE, poller thread should be signalled > immediately > +* > * RETURN VALUES > * None. > * > diff --git a/opensm/osm_req.c b/opensm/osm_req.c > index 7e9e664..3728327 100644 > --- a/opensm/osm_req.c > +++ b/opensm/osm_req.c > @@ -117,7 +117,7 @@ ib_api_status_t osm_req_get(IN osm_sm_t * sm, IN const > osm_dr_path_t * p_path, > if (p_context) > p_madw->context = *p_context; > > - osm_vl15_post(sm->p_vl15, p_madw); > + osm_vl15_post(sm->p_vl15, p_madw, FALSE); > > Exit: > OSM_LOG_EXIT(sm->p_log); > @@ -189,7 +189,7 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const > osm_dr_path_t * p_path, > > memcpy(osm_madw_get_smp_ptr(p_madw)->data, p_payload, payload_size); > > - osm_vl15_post(sm->p_vl15, p_madw); > + osm_vl15_post(sm->p_vl15, p_madw, FALSE); > > Exit: > OSM_LOG_EXIT(sm->p_log); > @@ -267,7 +267,7 @@ int osm_send_trap144(osm_sm_t * sm, ib_net16_t local) > "Sending Trap 144, TID 0x%" PRIx64 " to SM lid %u\n", > cl_ntoh64(smp->trans_id), cl_ntoh16(madw->mad_addr.dest_lid)); > > - osm_vl15_post(sm->p_vl15, madw); > + osm_vl15_post(sm->p_vl15, madw, FALSE); > > return 0; > } > diff --git a/opensm/osm_resp.c b/opensm/osm_resp.c > index ee391b6..cc52b68 100644 > --- a/opensm/osm_resp.c > +++ b/opensm/osm_resp.c > @@ -95,7 +95,8 @@ Exit: > ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > IN const osm_madw_t * p_req_madw, > IN ib_net16_t mad_status, > - IN const uint8_t * p_payload) > + IN const uint8_t * p_payload, > + IN boolean_t force_flag) > { > const ib_smp_t *p_req_smp; > ib_smp_t *p_smp; > @@ -142,7 +143,7 @@ ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > ib_get_sm_attr_str(p_smp->attr_id), cl_ntoh16(p_smp->attr_id), > cl_ntoh32(p_smp->attr_mod), cl_ntoh64(p_smp->trans_id)); > > - osm_vl15_post(sm->p_vl15, p_madw); > + osm_vl15_post(sm->p_vl15, p_madw, force_flag); > > Exit: > OSM_LOG_EXIT(sm->p_log); > diff --git a/opensm/osm_sm_mad_ctrl.c b/opensm/osm_sm_mad_ctrl.c > index f0bcff2..2ccfbfa 100644 > --- a/opensm/osm_sm_mad_ctrl.c > +++ b/opensm/osm_sm_mad_ctrl.c > @@ -163,7 +163,7 @@ static void sm_mad_ctrl_update_wire_stats(IN > osm_sm_mad_ctrl_t * p_ctrl) > We can signal the VL15 controller to send another MAD > if any are waiting for transmission. > */ > - osm_vl15_poll(p_ctrl->p_vl15); > + osm_vl15_poll(p_ctrl->p_vl15, FALSE); > OSM_LOG_EXIT(p_ctrl->p_log); > } > > diff --git a/opensm/osm_sminfo_rcv.c b/opensm/osm_sminfo_rcv.c > index e45d14e..6f0b099 100644 > --- a/opensm/osm_sminfo_rcv.c > +++ b/opensm/osm_sminfo_rcv.c > @@ -111,7 +111,7 @@ static void smi_rcv_process_get_request(IN osm_sm_t * sm, > p_smi->sm_key = 0; > } > > - status = osm_resp_send(sm, p_madw, 0, payload); > + status = osm_resp_send(sm, p_madw, 0, payload, TRUE); > if (status != IB_SUCCESS) { > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F02: " > "Error sending response (%s)\n", > @@ -208,7 +208,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * sm, > "Check legality failed. AttributeModifier:0x%X > RemoteState:%s\n", > p_smp->attr_mod, > osm_get_sm_mgr_state_str(ib_sminfo_get_state(sm_smi))); > - status = osm_resp_send(sm, p_madw, 7, payload); > + status = osm_resp_send(sm, p_madw, 7, payload, TRUE); > if (status != IB_SUCCESS) > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F05: " > "Error sending response (%s)\n", > @@ -253,7 +253,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * sm, > "AttributeModifier:0x%X RemoteState:%s\n", > p_smp->attr_mod, > osm_get_sm_mgr_state_str(ib_sminfo_get_state(sm_smi))); > - status = osm_resp_send(sm, p_madw, 7, payload); > + status = osm_resp_send(sm, p_madw, 7, payload, TRUE); > if (status != IB_SUCCESS) > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F08: " > "Error sending response (%s)\n", > @@ -263,7 +263,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * sm, > } > > /* the SubnSet(SMInfo) command is ok. Send a response. */ > - status = osm_resp_send(sm, p_madw, 0, payload); > + status = osm_resp_send(sm, p_madw, 0, payload, TRUE); > if (status != IB_SUCCESS) > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F09: " > "Error sending response (%s)\n", > diff --git a/opensm/osm_trap_rcv.c b/opensm/osm_trap_rcv.c > index 2aaaba4..132c510 100644 > --- a/opensm/osm_trap_rcv.c > +++ b/opensm/osm_trap_rcv.c > @@ -410,7 +410,7 @@ static void trap_rcv_process_request(IN osm_sm_t * sm, > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3809: " > "Failed to find source physical port for trap\n"); > > - status = osm_resp_send(sm, &tmp_madw, 0, payload); > + status = osm_resp_send(sm, &tmp_madw, 0, payload, FALSE); > if (status != IB_SUCCESS) { > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3802: " > "Error sending response (%s)\n", > diff --git a/opensm/osm_vl15intf.c b/opensm/osm_vl15intf.c > index c845597..5019f64 100644 > --- a/opensm/osm_vl15intf.c > +++ b/opensm/osm_vl15intf.c > @@ -131,9 +131,12 @@ static void vl15_poller(IN void *p_ptr) > The unicast FIFO has priority, since somebody is waiting > for a timely response. > */ > +send_next: > cl_spinlock_acquire(&p_vl->lock); > > - if (cl_qlist_count(&p_vl->ufifo) != 0) > + if (cl_qlist_count(&p_vl->hp_ufifo) != 0) > + p_fifo = &p_vl->hp_ufifo; > + else if (cl_qlist_count(&p_vl->ufifo) != 0) > p_fifo = &p_vl->ufifo; > else > p_fifo = &p_vl->rfifo; > @@ -151,6 +154,9 @@ static void vl15_poller(IN void *p_ptr) > OSM_LOG_FRAMES); > > vl15_send_mad(p_vl, p_madw); > + /* we should always drain the high priority queue first > */ > + if (p_fifo == &p_vl->hp_ufifo) > + goto send_next; > } else > /* > The VL15 FIFO is empty, so we have nothing left to > do. > @@ -195,6 +201,7 @@ void osm_vl15_construct(IN osm_vl15_t * p_vl) > cl_spinlock_construct(&p_vl->lock); > cl_qlist_init(&p_vl->rfifo); > cl_qlist_init(&p_vl->ufifo); > + cl_qlist_init(&p_vl->hp_ufifo); > cl_thread_construct(&p_vl->poller); > } > > @@ -232,6 +239,10 @@ void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct > osm_mad_pool *p_pool) > p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo); > osm_mad_pool_put(p_pool, p_madw); > } > + while (!cl_is_qlist_empty(&p_vl->hp_ufifo)) { > + p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo); > + osm_mad_pool_put(p_pool, p_madw); > + } > > cl_spinlock_release(&p_vl->lock); > > @@ -281,7 +292,7 @@ Exit: > return status; > } > > -void osm_vl15_poll(IN osm_vl15_t * p_vl) > +void osm_vl15_poll(IN osm_vl15_t * p_vl, IN boolean_t force_flag) > { > OSM_LOG_ENTER(p_vl->p_log); > > @@ -296,8 +307,8 @@ void osm_vl15_poll(IN osm_vl15_t * p_vl) > the event here. To cover this rare case, the poller > thread checks for a spurious wake-up. > */ > - if (p_vl->p_stats->qp0_mads_outstanding_on_wire < > - (int32_t) p_vl->max_wire_smps) { > + if (force_flag || (p_vl->p_stats->qp0_mads_outstanding_on_wire < > + (int32_t) p_vl->max_wire_smps)) { > OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > "Signalling poller thread\n"); > cl_event_signal(&p_vl->signal); > @@ -306,7 +317,7 @@ void osm_vl15_poll(IN osm_vl15_t * p_vl) > OSM_LOG_EXIT(p_vl->p_log); > } > > -void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw) > +void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw, boolean_t > force_flag) > { > OSM_LOG_ENTER(p_vl->p_log); > > @@ -322,7 +333,8 @@ void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * > p_madw) > cl_qlist_insert_tail(&p_vl->rfifo, &p_madw->list_item); > osm_stats_inc_qp0_outstanding(p_vl->p_stats); > } else > - cl_qlist_insert_tail(&p_vl->ufifo, &p_madw->list_item); > + cl_qlist_insert_tail(force_flag ? &p_vl->hp_ufifo : > &p_vl->ufifo, > + &p_madw->list_item); > cl_spinlock_release(&p_vl->lock); > > OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > @@ -330,7 +342,7 @@ void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * > p_madw) > p_vl->p_stats->qp0_mads_outstanding_on_wire, > p_vl->p_stats->qp0_mads_outstanding); > > - osm_vl15_poll(p_vl); > + osm_vl15_poll(p_vl, force_flag); > > OSM_LOG_EXIT(p_vl->p_log); > } > @@ -350,6 +362,16 @@ void osm_vl15_shutdown(IN osm_vl15_t * p_vl, IN > osm_mad_pool_t * p_mad_pool) > /* go over all outstanding MADs and retire their transactions */ > > /* first we handle the list of response MADs */ > + p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo); > + while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->hp_ufifo)) { > + OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > + "Releasing High Priority Response p_madw = %p\n", > p_madw); > + > + osm_mad_pool_put(p_mad_pool, p_madw); > + > + p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo); > + } > + > p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo); > while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->ufifo)) { > OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > -- > 1.7.7.6 > > > -- > > -- Alex > -- > To unsubscribe from this list: send the line "unsubscribe linux-rdma" in > the body of a message to [email protected] > More majordomo info at http://vger.kernel.org/majordomo-info.html -- Ira Weiny Member of Technical Staff Lawrence Livermore National Lab 925-423-8008 [email protected] -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
