Opensm relies on the libibumad layer not touching the input after it has been written to the kernel. Fix this by adding direct support for the umad address format. For the best performance, we integrate this into the kernel winmad driver. If the 'version' field is set, we use the winmad address format, otherwise, we assume umad address format.
The address format only matters when a GRH is present, which typically is not the case, and is not set by anything in tree. This patch avoids translating the address formats unless the grh present flag is set, which should provide a small performance improvement. Signed-off-by: Sean Hefty <[email protected]> --- trunk/core/winmad/kernel/wm_provider.c | 23 +++++++++++++++++----- trunk/core/winmad/wm_ioctl.h | 25 +++++++++++++++++++++++- trunk/inc/user/iba/winmad.h | 26 ++++++++++++++++++++++++- trunk/ulp/libibumad/include/infiniband/umad.h | 2 +- trunk/ulp/libibumad/src/umad.cpp | 13 +++---------- 5 files changed, 71 insertions(+), 18 deletions(-) diff --git a/trunk/core/winmad/kernel/wm_provider.c b/trunk/core/winmad/kernel/wm_provider.c index f3926f9..6e73c9d 100644 --- a/trunk/core/winmad/kernel/wm_provider.c +++ b/trunk/core/winmad/kernel/wm_provider.c @@ -284,6 +284,23 @@ out: WdfRequestCompleteWithInformation(Request, status, len); } +// If the Version is not set, use umad compatability address format +static void WmConvertGrh(ib_grh_t *pGrh, WM_IO_MAD *pIoMad) +{ + if (RtlUlongByteSwap(pIoMad->Address.VersionClassFlow) >> 28) { + pGrh->ver_class_flow = pIoMad->Address.VersionClassFlow; + } else { + pGrh->ver_class_flow = RtlUlongByteSwap((6 << 28) | + (((uint32_t) pIoMad->UmadAddress.TrafficClass) << 20) | + (pIoMad->UmadAddress.FlowLabel & 0x000FFFFF)); + } + + pGrh->hop_limit = pIoMad->Address.HopLimit; + // TODO: update IBAL to use SGID index + // pGrh->src_gid_index = pIoMad->Address.GidIndex; + RtlCopyMemory(pGrh->dest_gid.raw, pIoMad->Address.Gid, 16); +} + static NTSTATUS WmSendMad(WM_REGISTRATION *pRegistration, WM_IO_MAD *pIoMad, UINT32 size) { ib_al_ifc_t *pifc; @@ -305,11 +322,7 @@ static NTSTATUS WmSendMad(WM_REGISTRATION *pRegistration, WM_IO_MAD *pIoMad, UIN mad->retry_cnt = pIoMad->Retries; if ((mad->grh_valid = pIoMad->Address.GrhValid)) { - mad->p_grh->ver_class_flow = pIoMad->Address.VersionClassFlow; - mad->p_grh->hop_limit = pIoMad->Address.HopLimit; - // TODO: update IBAL to use SGID index - // mad->p_grh->src_gid_index = pIoMad->Address.GidIndex; - RtlCopyMemory(mad->p_grh->dest_gid.raw, pIoMad->Address.Gid, 16); + WmConvertGrh(mad->p_grh, pIoMad); } mad->remote_lid = pIoMad->Address.Lid; diff --git a/trunk/core/winmad/wm_ioctl.h b/trunk/core/winmad/wm_ioctl.h index 56063e0..6cdc220 100644 --- a/trunk/core/winmad/wm_ioctl.h +++ b/trunk/core/winmad/wm_ioctl.h @@ -98,6 +98,26 @@ typedef struct _WM_IO_MAD_AV } WM_IO_MAD_AV; +typedef struct _WM_IO_UMAD_AV +{ + NET32 Qpn; + NET32 Qkey; + UINT32 FlowLabel; + UINT16 PkeyIndex; + UINT8 HopLimit; + UINT8 GidIndex; + UINT8 Gid[16]; + + UINT8 TrafficClass; + UINT8 Reserved; + NET16 Lid; + UINT8 ServiceLevel; + UINT8 PathBits; + UINT8 StaticRate; + UINT8 GrhValid; + +} WM_IO_UMAD_AV; + #define WM_IO_SUCCESS 0 #define WM_IO_TIMEOUT 138 @@ -106,7 +126,10 @@ typedef struct _WM_IO_MAD_AV typedef struct _WM_IO_MAD { UINT64 Id; - WM_IO_MAD_AV Address; + union { + WM_IO_MAD_AV Address; + WM_IO_UMAD_AV UmadAddress; + }; UINT32 Status; UINT32 Timeout; diff --git a/trunk/inc/user/iba/winmad.h b/trunk/inc/user/iba/winmad.h index 35261f5..4f06173 100644 --- a/trunk/inc/user/iba/winmad.h +++ b/trunk/inc/user/iba/winmad.h @@ -72,6 +72,27 @@ typedef struct _WM_MAD_AV } WM_MAD_AV; +// Simplify casting to WM_MAD_AV +typedef struct _WM_UMAD_AV +{ + NET32 Qpn; + NET32 Qkey; + UINT32 FlowLabel; + UINT16 PkeyIndex; + UINT8 HopLimit; + UINT8 GidIndex; + UINT8 Gid[16]; + + UINT8 TrafficClass; + UINT8 ReservedGrh; + NET16 Lid; + UINT8 ServiceLevel; + UINT8 PathBits; + UINT8 StaticRate; + UINT8 GrhValid; + +} WM_UMAD_AV; + #define WM_SUCCESS 0 #define WM_TIMEOUT 138 @@ -80,7 +101,10 @@ typedef struct _WM_MAD_AV typedef struct _WM_MAD { UINT64 Id; - WM_MAD_AV Address; + union { + WM_MAD_AV Address; + WM_UMAD_AV UmadAddress; + }; UINT32 Status; UINT32 Timeout; diff --git a/trunk/ulp/libibumad/include/infiniband/umad.h b/trunk/ulp/libibumad/include/infiniband/umad.h index f6b41ac..aceebf3 100644 --- a/trunk/ulp/libibumad/include/infiniband/umad.h +++ b/trunk/ulp/libibumad/include/infiniband/umad.h @@ -52,7 +52,7 @@ typedef unsigned __int64 uint64_t; #define UMAD_MAX_DEVICES 20 #define UMAD_ANY_PORT 0 -// Allow casting to WM_MAD_AV +// Allow casting to WM_UMAD_AV typedef struct ib_mad_addr { uint32_t qpn; diff --git a/trunk/ulp/libibumad/src/umad.cpp b/trunk/ulp/libibumad/src/umad.cpp index d4759c7..19582dc 100644 --- a/trunk/ulp/libibumad/src/umad.cpp +++ b/trunk/ulp/libibumad/src/umad.cpp @@ -564,14 +564,6 @@ size_t umad_size(void) return sizeof(struct ib_user_mad); } -static void umad_convert_addr(struct ib_mad_addr *addr, WM_MAD_AV *av) -{ - av->VersionClassFlow = htonl((6 << 28) | (((uint32_t) addr->traffic_class) << 20) | - (addr->flow_label & 0x000FFFFF)); - av->Reserved = 0; - av->StaticRate = 0; -} - static void umad_convert_av(WM_MAD_AV *av, struct ib_mad_addr *addr) { uint32_t ver_class_flow; @@ -595,9 +587,10 @@ int umad_send(int portid, int agentid, void *umad, int length, mad->retries = (uint32_t) retries; mad->length = (uint32_t) length; - umad_convert_addr(&mad->addr, &((WM_MAD *) mad)->Address); + mad->addr.reserved_grh = 0; + mad->addr.reserved_rate = 0; + hr = ports[portid].prov->Send((WM_MAD *) mad, NULL); - umad_convert_av(&((WM_MAD *) mad)->Address, &mad->addr); if (FAILED(hr)) { _set_errno(EIO); return GetLastError(); _______________________________________________ ofw mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
