Connection establishment rates take a huge performance hit as a result of using FindPkey, which results in querying the HCA to lookup current pkey information. Add a cache to the userspace library for pkey information.
The cache is updated using the CWVDevice:Notify() routine. Before any access is made to the cache, the caller checks for changes to the partition tables. If a change is found, then the cache is updated. Otherwise, the cache is accessed to map pkey values to indices. This results in the connection rate over winverbs almost doubling. Signed-off-by: Sean Hefty <[email protected]> --- trunk/core/winverbs/user/wv_device.cpp | 149 +++++++++++++++++++++++++------- trunk/core/winverbs/user/wv_device.h | 20 ++++ 2 files changed, 137 insertions(+), 32 deletions(-) diff --git a/trunk/core/winverbs/user/wv_device.cpp b/trunk/core/winverbs/user/wv_device.cpp index 9aac788..b260d41 100644 --- a/trunk/core/winverbs/user/wv_device.cpp +++ b/trunk/core/winverbs/user/wv_device.cpp @@ -48,6 +48,8 @@ CWVDevice::CWVDevice(CWVProvider *pProvider) m_pProvider = pProvider; m_hFile = pProvider->m_hFile; + m_PortCount = 0; + m_pPorts = NULL; m_hVerbsDevice = NULL; m_hLib = NULL; } @@ -121,11 +123,56 @@ post: hr = WvConvertIbStatus(stat); } + if (SUCCEEDED(hr)) { + hr = InitPorts(); + } + + return hr; +} + +STDMETHODIMP CWVDevice:: +InitPorts() +{ + WV_DEVICE_ATTRIBUTES attr; + WV_PORT *port; + HRESULT hr; + + hr = Query(&attr); + if (FAILED(hr)) { + return hr; + } + + m_pPorts = new WV_PORT[attr.PhysPortCount]; + if (m_pPorts == NULL) { + return WV_NO_MEMORY; + } + + RtlZeroMemory(m_pPorts, sizeof(WV_PORT) * attr.PhysPortCount); + for (m_PortCount = 0; m_PortCount < attr.PhysPortCount; m_PortCount++) { + port = &m_pPorts[m_PortCount]; + + port->m_Overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + if (port->m_Overlap.hEvent == NULL) { + hr = WV_INSUFFICIENT_RESOURCES; + break; + } + + port->m_Overlap.hEvent = (HANDLE) ((ULONG_PTR) port->m_Overlap.hEvent | 1); + + port->m_Flags = 0xFFFFFFFF; + hr = UpdatePort(m_PortCount + 1); + if (FAILED(hr)) { + CloseHandle(port->m_Overlap.hEvent); + break; + } + } + return hr; } CWVDevice::~CWVDevice() { + WV_PORT *port; DWORD bytes; HRESULT hr; @@ -137,6 +184,16 @@ CWVDevice::~CWVDevice() m_Verbs.post_close_ca(m_hVerbsDevice, (ib_api_status_t) hr); } + while (m_PortCount--) { + port = &m_pPorts[m_PortCount]; + GetOverlappedResult(&port->m_Overlap, &bytes, TRUE); + CloseHandle(port->m_Overlap.hEvent); + } + + if (m_pPorts != NULL) { + delete m_pPorts; + } + if (m_hLib != NULL) { FreeLibrary(m_hLib); } @@ -252,6 +309,26 @@ QueryPort(UINT8 PortNumber, WV_PORT_ATTRIBUTES* pAttributes) return HRESULT_FROM_WIN32(GetLastError()); } + pAttributes->PkeyTableLength = min(pAttributes->PkeyTableLength, WV_MAX_PKEYS); + return WV_SUCCESS; +} + +STDMETHODIMP CWVDevice:: +UpdatePort(UINT8 PortNumber) +{ + WV_PORT *port = &m_pPorts[PortNumber - 1]; + HRESULT hr; + + if (port->m_Flags & WV_EVENT_PARTITION) { + UpdatePkeys(PortNumber); + } + + port->m_Flags = 0; + hr = Notify(PortNumber, &port->m_Overlap, &port->m_Flags); + if (FAILED(hr) && hr != WV_IO_PENDING) { + return hr; + } + return WV_SUCCESS; } @@ -316,63 +393,73 @@ FindGid(UINT8 PortNumber, WV_GID *pGid, DWORD *pIndex) } STDMETHODIMP CWVDevice:: -QueryPkey(UINT8 PortNumber, UINT16 Index, NET16* pPkey) +UpdatePkeys(UINT8 PortNumber) { WV_IO_DEVICE_PORT_QUERY query; - NET16 *pkeytable; - DWORD npkey, bytes; - HRESULT hr; - CWVBuffer buf; + WV_PORT *port; + DWORD bytes; - bytes = sizeof NET16 * (Index + 1); - pkeytable = (NET16 *) buf.Get(bytes); - if (pkeytable == NULL) { - return WV_NO_MEMORY; - } + port = &m_pPorts[PortNumber - 1]; + bytes = sizeof(port->m_PkeyTable); query.Id = m_Id; query.PortNumber = PortNumber; RtlZeroMemory(&query.Reserved, sizeof query.Reserved); if (!WvDeviceIoControl(m_hFile, WV_IOCTL_DEVICE_PKEY_QUERY, &query, - sizeof query, pkeytable, bytes, &bytes, NULL)) { - hr = HRESULT_FROM_WIN32(GetLastError()); - goto out; + sizeof query, port->m_PkeyTable, bytes, &bytes, NULL)) { + port->m_PkeyCount = 0; + return HRESULT_FROM_WIN32(GetLastError()); + } + + port->m_PkeyCount = (UINT16) (bytes / sizeof NET16); + return WV_SUCCESS; +} + +STDMETHODIMP CWVDevice:: +QueryPkey(UINT8 PortNumber, UINT16 Index, NET16* pPkey) +{ + WV_PORT *port = &m_pPorts[PortNumber - 1]; + HRESULT hr; + + EnterCriticalSection(&m_CritSec); + if (HasOverlappedIoCompleted(&port->m_Overlap)) { + UpdatePort(PortNumber); } - npkey = bytes / sizeof NET16; - if (Index >= npkey) { + if (Index < port->m_PkeyCount) { + *pPkey = port->m_PkeyTable[Index]; + hr = WV_SUCCESS; + } else { hr = WV_INVALID_PARAMETER_2; - goto out; } - *pPkey = pkeytable[Index]; - hr = WV_SUCCESS; -out: - buf.Put(); + LeaveCriticalSection(&m_CritSec); return hr; } STDMETHODIMP CWVDevice:: FindPkey(UINT8 PortNumber, NET16 Pkey, UINT16 *pIndex) { - NET16 key; + WV_PORT *port = &m_pPorts[PortNumber - 1]; UINT16 index; - HRESULT hr; + HRESULT hr = WV_INVALID_ADDRESS; - for (index = 0; true; index++) { - hr = QueryPkey(PortNumber, index, &key); - if (FAILED(hr)) { - return hr; - } + EnterCriticalSection(&m_CritSec); + if (HasOverlappedIoCompleted(&port->m_Overlap)) { + UpdatePort(PortNumber); + } - if (Pkey == key) { - *pIndex = (UINT16) index; - return WV_SUCCESS; + for (index = 0; index < port->m_PkeyCount; index++) { + if (Pkey == port->m_PkeyTable[index]) { + *pIndex = index; + hr = WV_SUCCESS; + break; } } - return WV_INVALID_ADDRESS; + LeaveCriticalSection(&m_CritSec); + return hr; } STDMETHODIMP CWVDevice:: diff --git a/trunk/core/winverbs/user/wv_device.h b/trunk/core/winverbs/user/wv_device.h index 7e98da1..9f4009c 100644 --- a/trunk/core/winverbs/user/wv_device.h +++ b/trunk/core/winverbs/user/wv_device.h @@ -38,6 +38,18 @@ #include "wv_provider.h" #include "wv_base.h" +#define WV_MAX_PKEYS 16 + +typedef struct _WV_PORT +{ + OVERLAPPED m_Overlap; + DWORD m_Flags; + UINT16 m_PkeyCount; + NET16 m_PkeyTable[WV_MAX_PKEYS]; + +} WV_PORT; + + class CWVDevice : IWVDevice, public CWVBase { public: @@ -105,7 +117,13 @@ public: protected: HMODULE m_hLib; - STDMETHODIMP Open(NET64 Guid); + STDMETHODIMP Open(NET64 Guid); + + WV_PORT *m_pPorts; + UINT8 m_PortCount; + STDMETHODIMP InitPorts(); + STDMETHODIMP UpdatePort(UINT8 PortNumber); + STDMETHODIMP UpdatePkeys(UINT8 PortNumber); }; #endif // __WV_DEVICE_H_ _______________________________________________ ofw mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
