Please ignore for now. A new patch will follow. On 10:51 Thu 24 May , Ira Weiny wrote: > On Thu, 24 May 2012 15:10:23 +0300 > Alex Netes <[email protected]> wrote: > > > The issue is resolved by adding High priority unicast FIFO queue to the VL15 > > poller thread. > > > > Signed-off-by: Vladimir Koushnir <[email protected]> > > Signed-off-by: Alex Netes <[email protected]> > > --- > > include/opensm/osm_sm.h | 7 +++++-- > > include/opensm/osm_vl15intf.h | 15 +++++++++++++-- > > opensm/osm_req.c | 6 +++--- > > opensm/osm_resp.c | 5 +++-- > > opensm/osm_sm_mad_ctrl.c | 2 +- > > opensm/osm_sminfo_rcv.c | 8 ++++---- > > opensm/osm_trap_rcv.c | 2 +- > > opensm/osm_vl15intf.c | 36 +++++++++++++++++++++++++++++------- > > 8 files changed, 59 insertions(+), 22 deletions(-) > > > > diff --git a/include/opensm/osm_sm.h b/include/opensm/osm_sm.h > > index 4c7b120..17d6afd 100644 > > --- a/include/opensm/osm_sm.h > > +++ b/include/opensm/osm_sm.h > > @@ -495,7 +495,8 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const > > osm_dr_path_t * p_path, > > ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > > IN const osm_madw_t * p_req_madw, > > IN ib_net16_t status, > > - IN const uint8_t * p_payload); > > + IN const uint8_t * p_payload, > > + IN boolean_t force_flag); > > /* > > * PARAMETERS > > * p_resp > > @@ -510,7 +511,9 @@ ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > > * > > * p_payload > > * [in] Pointer to the payload of the response MAD. > > -* > > +* force_flag > > +* [in] If TRUE, indicates that the response should be processed > > immediately > > +* by VL15 poller thread > > Shouldn't "force_flag" be something like "immediate"? Same comment for all > callers of this function. > > > * RETURN VALUES > > * IB_SUCCESS if the response was successful. > > * > > diff --git a/include/opensm/osm_vl15intf.h b/include/opensm/osm_vl15intf.h > > index e621c68..a600de6 100644 > > --- a/include/opensm/osm_vl15intf.h > > +++ b/include/opensm/osm_vl15intf.h > > @@ -123,6 +123,7 @@ typedef struct osm_vl15 { > > cl_thread_t poller; > > cl_qlist_t rfifo; > > cl_qlist_t ufifo; > > + cl_qlist_t hp_ufifo; > > cl_spinlock_t lock; > > osm_vendor_t *p_vend; > > osm_log_t *p_log; > > @@ -156,6 +157,10 @@ typedef struct osm_vl15 { > > * First-in First-out queue for outbound VL15 MADs for which > > * no response is expected, aka the "unicast fifo". > > * > > +* hp_ufifo > > +* First-in First-out high priority queue for outbound VL15 MADs > > for which > > +* no response is expected. SMInfo response MADs are placed in > > this queue > > +* > > * poller > > * Worker thread pool that services the fifo to transmit VL15 MADs > > * > > @@ -298,7 +303,7 @@ ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl15, > > IN osm_vendor_t * p_vend, > > * > > * SYNOPSIS > > */ > > -void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw); > > +void osm_vl15_post(IN osm_vl15_t * p_vl15, IN osm_madw_t * p_madw, IN > > boolean_t force_flag); > > /* > > * PARAMETERS > > * p_vl15 > > @@ -307,6 +312,9 @@ void osm_vl15_post(IN osm_vl15_t * p_vl15, IN > > osm_madw_t * p_madw); > > * p_madw > > * [in] Pointer to a MAD wrapper structure containing the MAD. > > * > > +* force_flag > > +* [in] If set to TRUE, high priority ucast queue should be used > > +* > > or "high_priority"? Same comment for all callers of this function. > > Ira > > > * RETURN VALUES > > * This function does not return a value. > > * > > @@ -327,12 +335,15 @@ void osm_vl15_post(IN osm_vl15_t * p_vl15, IN > > osm_madw_t * p_madw); > > * > > * SYNOPSIS > > */ > > -void osm_vl15_poll(IN osm_vl15_t * p_vl); > > +void osm_vl15_poll(IN osm_vl15_t * p_vl, IN boolean_t force_flag); > > /* > > * PARAMETERS > > * p_vl15 > > * [in] Pointer to an osm_vl15_t object. > > * > > +* force_flag > > +* [in] If set to TRUE, poller thread should be signalled > > immediately > > +* > > * RETURN VALUES > > * None. > > * > > diff --git a/opensm/osm_req.c b/opensm/osm_req.c > > index 7e9e664..3728327 100644 > > --- a/opensm/osm_req.c > > +++ b/opensm/osm_req.c > > @@ -117,7 +117,7 @@ ib_api_status_t osm_req_get(IN osm_sm_t * sm, IN const > > osm_dr_path_t * p_path, > > if (p_context) > > p_madw->context = *p_context; > > > > - osm_vl15_post(sm->p_vl15, p_madw); > > + osm_vl15_post(sm->p_vl15, p_madw, FALSE); > > > > Exit: > > OSM_LOG_EXIT(sm->p_log); > > @@ -189,7 +189,7 @@ ib_api_status_t osm_req_set(IN osm_sm_t * sm, IN const > > osm_dr_path_t * p_path, > > > > memcpy(osm_madw_get_smp_ptr(p_madw)->data, p_payload, payload_size); > > > > - osm_vl15_post(sm->p_vl15, p_madw); > > + osm_vl15_post(sm->p_vl15, p_madw, FALSE); > > > > Exit: > > OSM_LOG_EXIT(sm->p_log); > > @@ -267,7 +267,7 @@ int osm_send_trap144(osm_sm_t * sm, ib_net16_t local) > > "Sending Trap 144, TID 0x%" PRIx64 " to SM lid %u\n", > > cl_ntoh64(smp->trans_id), cl_ntoh16(madw->mad_addr.dest_lid)); > > > > - osm_vl15_post(sm->p_vl15, madw); > > + osm_vl15_post(sm->p_vl15, madw, FALSE); > > > > return 0; > > } > > diff --git a/opensm/osm_resp.c b/opensm/osm_resp.c > > index ee391b6..cc52b68 100644 > > --- a/opensm/osm_resp.c > > +++ b/opensm/osm_resp.c > > @@ -95,7 +95,8 @@ Exit: > > ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > > IN const osm_madw_t * p_req_madw, > > IN ib_net16_t mad_status, > > - IN const uint8_t * p_payload) > > + IN const uint8_t * p_payload, > > + IN boolean_t force_flag) > > { > > const ib_smp_t *p_req_smp; > > ib_smp_t *p_smp; > > @@ -142,7 +143,7 @@ ib_api_status_t osm_resp_send(IN osm_sm_t * sm, > > ib_get_sm_attr_str(p_smp->attr_id), cl_ntoh16(p_smp->attr_id), > > cl_ntoh32(p_smp->attr_mod), cl_ntoh64(p_smp->trans_id)); > > > > - osm_vl15_post(sm->p_vl15, p_madw); > > + osm_vl15_post(sm->p_vl15, p_madw, force_flag); > > > > Exit: > > OSM_LOG_EXIT(sm->p_log); > > diff --git a/opensm/osm_sm_mad_ctrl.c b/opensm/osm_sm_mad_ctrl.c > > index f0bcff2..2ccfbfa 100644 > > --- a/opensm/osm_sm_mad_ctrl.c > > +++ b/opensm/osm_sm_mad_ctrl.c > > @@ -163,7 +163,7 @@ static void sm_mad_ctrl_update_wire_stats(IN > > osm_sm_mad_ctrl_t * p_ctrl) > > We can signal the VL15 controller to send another MAD > > if any are waiting for transmission. > > */ > > - osm_vl15_poll(p_ctrl->p_vl15); > > + osm_vl15_poll(p_ctrl->p_vl15, FALSE); > > OSM_LOG_EXIT(p_ctrl->p_log); > > } > > > > diff --git a/opensm/osm_sminfo_rcv.c b/opensm/osm_sminfo_rcv.c > > index e45d14e..6f0b099 100644 > > --- a/opensm/osm_sminfo_rcv.c > > +++ b/opensm/osm_sminfo_rcv.c > > @@ -111,7 +111,7 @@ static void smi_rcv_process_get_request(IN osm_sm_t * > > sm, > > p_smi->sm_key = 0; > > } > > > > - status = osm_resp_send(sm, p_madw, 0, payload); > > + status = osm_resp_send(sm, p_madw, 0, payload, TRUE); > > if (status != IB_SUCCESS) { > > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F02: " > > "Error sending response (%s)\n", > > @@ -208,7 +208,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * > > sm, > > "Check legality failed. AttributeModifier:0x%X > > RemoteState:%s\n", > > p_smp->attr_mod, > > osm_get_sm_mgr_state_str(ib_sminfo_get_state(sm_smi))); > > - status = osm_resp_send(sm, p_madw, 7, payload); > > + status = osm_resp_send(sm, p_madw, 7, payload, TRUE); > > if (status != IB_SUCCESS) > > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F05: " > > "Error sending response (%s)\n", > > @@ -253,7 +253,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * > > sm, > > "AttributeModifier:0x%X RemoteState:%s\n", > > p_smp->attr_mod, > > osm_get_sm_mgr_state_str(ib_sminfo_get_state(sm_smi))); > > - status = osm_resp_send(sm, p_madw, 7, payload); > > + status = osm_resp_send(sm, p_madw, 7, payload, TRUE); > > if (status != IB_SUCCESS) > > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F08: " > > "Error sending response (%s)\n", > > @@ -263,7 +263,7 @@ static void smi_rcv_process_set_request(IN osm_sm_t * > > sm, > > } > > > > /* the SubnSet(SMInfo) command is ok. Send a response. */ > > - status = osm_resp_send(sm, p_madw, 0, payload); > > + status = osm_resp_send(sm, p_madw, 0, payload, TRUE); > > if (status != IB_SUCCESS) > > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 2F09: " > > "Error sending response (%s)\n", > > diff --git a/opensm/osm_trap_rcv.c b/opensm/osm_trap_rcv.c > > index 2aaaba4..132c510 100644 > > --- a/opensm/osm_trap_rcv.c > > +++ b/opensm/osm_trap_rcv.c > > @@ -410,7 +410,7 @@ static void trap_rcv_process_request(IN osm_sm_t * sm, > > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3809: " > > "Failed to find source physical port for trap\n"); > > > > - status = osm_resp_send(sm, &tmp_madw, 0, payload); > > + status = osm_resp_send(sm, &tmp_madw, 0, payload, FALSE); > > if (status != IB_SUCCESS) { > > OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3802: " > > "Error sending response (%s)\n", > > diff --git a/opensm/osm_vl15intf.c b/opensm/osm_vl15intf.c > > index c845597..5019f64 100644 > > --- a/opensm/osm_vl15intf.c > > +++ b/opensm/osm_vl15intf.c > > @@ -131,9 +131,12 @@ static void vl15_poller(IN void *p_ptr) > > The unicast FIFO has priority, since somebody is waiting > > for a timely response. > > */ > > +send_next: > > cl_spinlock_acquire(&p_vl->lock); > > > > - if (cl_qlist_count(&p_vl->ufifo) != 0) > > + if (cl_qlist_count(&p_vl->hp_ufifo) != 0) > > + p_fifo = &p_vl->hp_ufifo; > > + else if (cl_qlist_count(&p_vl->ufifo) != 0) > > p_fifo = &p_vl->ufifo; > > else > > p_fifo = &p_vl->rfifo; > > @@ -151,6 +154,9 @@ static void vl15_poller(IN void *p_ptr) > > OSM_LOG_FRAMES); > > > > vl15_send_mad(p_vl, p_madw); > > + /* we should always drain the high priority queue first > > */ > > + if (p_fifo == &p_vl->hp_ufifo) > > + goto send_next; > > } else > > /* > > The VL15 FIFO is empty, so we have nothing left to > > do. > > @@ -195,6 +201,7 @@ void osm_vl15_construct(IN osm_vl15_t * p_vl) > > cl_spinlock_construct(&p_vl->lock); > > cl_qlist_init(&p_vl->rfifo); > > cl_qlist_init(&p_vl->ufifo); > > + cl_qlist_init(&p_vl->hp_ufifo); > > cl_thread_construct(&p_vl->poller); > > } > > > > @@ -232,6 +239,10 @@ void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct > > osm_mad_pool *p_pool) > > p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo); > > osm_mad_pool_put(p_pool, p_madw); > > } > > + while (!cl_is_qlist_empty(&p_vl->hp_ufifo)) { > > + p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo); > > + osm_mad_pool_put(p_pool, p_madw); > > + } > > > > cl_spinlock_release(&p_vl->lock); > > > > @@ -281,7 +292,7 @@ Exit: > > return status; > > } > > > > -void osm_vl15_poll(IN osm_vl15_t * p_vl) > > +void osm_vl15_poll(IN osm_vl15_t * p_vl, IN boolean_t force_flag) > > { > > OSM_LOG_ENTER(p_vl->p_log); > > > > @@ -296,8 +307,8 @@ void osm_vl15_poll(IN osm_vl15_t * p_vl) > > the event here. To cover this rare case, the poller > > thread checks for a spurious wake-up. > > */ > > - if (p_vl->p_stats->qp0_mads_outstanding_on_wire < > > - (int32_t) p_vl->max_wire_smps) { > > + if (force_flag || (p_vl->p_stats->qp0_mads_outstanding_on_wire < > > + (int32_t) p_vl->max_wire_smps)) { > > OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > > "Signalling poller thread\n"); > > cl_event_signal(&p_vl->signal); > > @@ -306,7 +317,7 @@ void osm_vl15_poll(IN osm_vl15_t * p_vl) > > OSM_LOG_EXIT(p_vl->p_log); > > } > > > > -void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw) > > +void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw, boolean_t > > force_flag) > > { > > OSM_LOG_ENTER(p_vl->p_log); > > > > @@ -322,7 +333,8 @@ void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t > > * p_madw) > > cl_qlist_insert_tail(&p_vl->rfifo, &p_madw->list_item); > > osm_stats_inc_qp0_outstanding(p_vl->p_stats); > > } else > > - cl_qlist_insert_tail(&p_vl->ufifo, &p_madw->list_item); > > + cl_qlist_insert_tail(force_flag ? &p_vl->hp_ufifo : > > &p_vl->ufifo, > > + &p_madw->list_item); > > cl_spinlock_release(&p_vl->lock); > > > > OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > > @@ -330,7 +342,7 @@ void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t > > * p_madw) > > p_vl->p_stats->qp0_mads_outstanding_on_wire, > > p_vl->p_stats->qp0_mads_outstanding); > > > > - osm_vl15_poll(p_vl); > > + osm_vl15_poll(p_vl, force_flag); > > > > OSM_LOG_EXIT(p_vl->p_log); > > } > > @@ -350,6 +362,16 @@ void osm_vl15_shutdown(IN osm_vl15_t * p_vl, IN > > osm_mad_pool_t * p_mad_pool) > > /* go over all outstanding MADs and retire their transactions */ > > > > /* first we handle the list of response MADs */ > > + p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo); > > + while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->hp_ufifo)) { > > + OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > > + "Releasing High Priority Response p_madw = %p\n", > > p_madw); > > + > > + osm_mad_pool_put(p_mad_pool, p_madw); > > + > > + p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->hp_ufifo); > > + } > > + > > p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo); > > while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->ufifo)) { > > OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, > > -- > > 1.7.7.6 > > > > > > -- > > > > -- Alex > > -- > > To unsubscribe from this list: send the line "unsubscribe linux-rdma" in > > the body of a message to [email protected] > > More majordomo info at http://vger.kernel.org/majordomo-info.html > > > -- > Ira Weiny > Member of Technical Staff > Lawrence Livermore National Lab > 925-423-8008 > [email protected] > -- > To unsubscribe from this list: send the line "unsubscribe linux-rdma" in > the body of a message to [email protected] > More majordomo info at http://vger.kernel.org/majordomo-info.html
-- -- Alex -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
