Galina caught an issue and reported it to me offlist: the ND provider itself needs some tweaks for this to work, because the connector logic was passing the port number down to the kernel drivers in host order. Additionally, the listen path was not swapping the SID to network order.
In this patch, I added helper routines in ib_types.h to format and decode a RDMA_CM service ID, and use those helper routines throughout the ND provider for consistency. Signed-off-by: Fab Tillier <[email protected]> Index: ulp/nd/user/NdConnector.cpp =================================================================== --- ulp/nd/user/NdConnector.cpp (revision 3095) +++ ulp/nd/user/NdConnector.cpp (working copy) @@ -306,7 +306,7 @@ HRESULT CConnector::Connect( { m_LocalPort = LocalPort; } - ioctl.pdata.src_port = _byteswap_ushort( m_LocalPort ); + ioctl.pdata.src_port = m_LocalPort; // Resolve the GIDs. HRESULT hr = IBAT::Resolve( Index: ulp/nd/user/NdListen.cpp =================================================================== --- ulp/nd/user/NdListen.cpp (revision 3095) +++ ulp/nd/user/NdListen.cpp (working copy) @@ -187,8 +187,7 @@ HRESULT GetPdataForActive( ual_cep_listen_ioctl_t listen; listen.cid = 0; - listen.cep_listen.svc_id = - 0x0000000001000000 | Protocol << 16 | Port; + listen.cep_listen.svc_id = ib_cm_rdma_sid( Protocol, Port ); listen.cep_listen.port_guid = m_pParent->m_PortGuid; @@ -244,7 +243,9 @@ HRESULT GetPdataForActive( ND_PRINT( TRACE_LEVEL_INFORMATION, ND_DBG_NDI, ("Created listen CEP with cid %d \n", m_cid ) ); - // TODO: Come up with something better for port number. + // + // The following port calculation must match what is done in the kernel. + // if( Port == 0 ) Port = (USHORT)m_cid | (USHORT)(m_cid >> 16); Index: core/al/kernel/al_cm_cep.c =================================================================== --- core/al/kernel/al_cm_cep.c (revision 3095) +++ core/al/kernel/al_cm_cep.c (working copy) @@ -3390,7 +3390,7 @@ __cep_queue_mad( // TODO: Remove - manage above core kernel CM code /* NDI connection request case */ if ( p_cep->state == CEP_STATE_LISTEN && - (p_cep->sid & ~0x0ffffffI64) == IB_REQ_CM_RDMA_SID_PREFIX ) + (p_cep->sid & IB_REQ_CM_RDMA_SID_PREFIX_MASK) == IB_REQ_CM_RDMA_SID_PREFIX ) { /* Try to complete pending IRP, if any */ mad_cm_req_t* p_req = (mad_cm_req_t*)ib_get_mad_buf( p_mad ); ib_cm_rdma_req_t *p_rdma_req = (ib_cm_rdma_req_t *)p_req->pdata; @@ -3401,7 +3401,7 @@ __cep_queue_mad( (p_rdma_req->ipv != 0x40 && p_rdma_req->ipv != 0x60) ) { AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, - ("NDI connection req is rejected: maj_min_ver %d, ipv %#x \n", + ("RDMA CM connection req is rejected: maj_min_ver %d, ipv %#x \n", p_rdma_req->maj_min_ver, p_rdma_req->ipv ) ); return IB_UNSUPPORTED; } Index: core/al/kernel/al_ndi_cm.c =================================================================== --- core/al/kernel/al_ndi_cm.c (revision 3095) +++ core/al/kernel/al_ndi_cm.c (working copy) @@ -1061,6 +1061,7 @@ nd_cm_handler( AL_EXIT( AL_DBG_NDI ); } + static void __ndi_fill_cm_req( IN net32_t qpn, @@ -1072,7 +1073,7 @@ __ndi_fill_cm_req( memset( p_cm_req, 0, sizeof(*p_cm_req) ); - p_cm_req->service_id = IB_REQ_CM_RDMA_SID_PREFIX | (p_req->prot << 16) | p_req->dst_port; + p_cm_req->service_id = ib_cm_rdma_sid( p_req->prot, p_req->dst_port ); p_cm_req->p_primary_path = p_path_rec; p_cm_req->qpn = qpn; @@ -1922,9 +1923,12 @@ ndi_listen_cm( p_csq->state = NDI_CM_LISTEN; __ndi_release_lock( &p_csq->csq, irql ); - if( (p_listen->svc_id & 0xFFFF) == 0 ) + if( ib_cm_rdma_sid_port( p_listen->svc_id ) == 0 ) { - p_listen->svc_id |= (USHORT)cid | (USHORT)(cid >> 16); + p_listen->svc_id = ib_cm_rdma_sid( + ib_cm_rdma_sid_protocol( p_listen->svc_id ), + (USHORT)cid | (USHORT)(cid >> 16) + ); } ib_status = al_cep_listen( h_al, cid, p_listen ); Index: inc/iba/ib_types.h =================================================================== --- inc/iba/ib_types.h (revision 3095) +++ inc/iba/ib_types.h (working copy) @@ -12472,10 +12472,34 @@ typedef struct _ib_time_stamp { * ib_cc_mad_t *********/ -#define IB_REQ_CM_RDMA_SID_PREFIX 0x0000000001000000 +#define IB_REQ_CM_RDMA_SID_PREFIX CL_NTOH64( 0x0000000001000000I64 ) +#define IB_REQ_CM_RDMA_SID_PREFIX_MASK CL_NTOH64( 0xFFFFFFFFFF000000I64 ) #define IB_REQ_CM_RDMA_PDATA_SIZE 56 #define IB_REQ_CM_RDMA_MAJOR_VERSION 0 #define IB_REQ_CM_RDMA_MINOR_VERSION 0 + +AL_INLINE net64_t AL_API +ib_cm_rdma_sid( + IN uint8_t protocol, + IN net16_t port ) +{ + return IB_REQ_CM_RDMA_SID_PREFIX | ((UINT64)protocol) << 40 | ((UINT64)port) << 48; +} + +AL_INLINE net16_t AL_API +ib_cm_rdma_sid_port( + IN net64_t sid ) +{ + return (net16_t)(sid >> 48); +} + +AL_INLINE uint8_t AL_API +ib_cm_rdma_sid_protocol( + IN net64_t sid ) +{ + return (uint8_t)(sid >> 40); +} + /****s* Access Layer/ib_cm_rep_t Index: inc/iba/ib_al_ioctl.h =================================================================== --- inc/iba/ib_al_ioctl.h (revision 3095) +++ inc/iba/ib_al_ioctl.h (working copy) @@ -3479,7 +3479,7 @@ typedef struct _ual_ndi_req_cm_ioctl_in uint64_t h_qp; net64_t guid; net32_t cid; - uint16_t dst_port; + net16_t dst_port; uint8_t resp_res; uint8_t init_depth; uint8_t prot;
nd_sid.2.patch
Description: nd_sid.2.patch
_______________________________________________ ofw mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
