Ok. Can you describe what the problem is that is being worked around? Thanks, -Fab
From: Reuven Amitai [mailto:[EMAIL PROTECTED] Sent: Wednesday, April 09, 2008 10:14 AM To: Fab Tillier; [email protected] Subject: RE: [PATCH] mtcr Workaround. ________________________________ From: Fab Tillier [mailto:[EMAIL PROTECTED] Sent: Wednesday, April 09, 2008 7:54 PM To: Reuven Amitai; [email protected] Subject: RE: [PATCH] mtcr What is WA? From: [EMAIL PROTECTED] [mailto:[EMAIL PROTECTED] On Behalf Of Reuven Amitai Sent: Wednesday, April 09, 2008 6:27 AM To: [email protected] Subject: [ofw] [PATCH] mtcr Hi, The following patch adds WA for Hermon chip. Thanks, Reuven. Index: tools/mtcr/user/mtcr.c =================================================================== --- tools/mtcr/user/mtcr.c (revision 1047) +++ tools/mtcr/user/mtcr.c (working copy) @@ -21,33 +21,82 @@ #define USB_DEV_NAME "mtusb-1" #define CLEAR(st) memset(&(st), 0, sizeof(st)) +// HERMON ORDERING WA: +#define HERMON_WA_BASE 0xf0384 // SEM BASE ADDR. SEM 0xf0380 is reserved for external tools usage. +#define HERMON_WA_SIZE 3 // Size in entries +#define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL" -#define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL" +/* + * DLL global variables + */ +#pragma data_seg(".sdata") + +/* Process / thread count */ +DWORD g_Slots[HERMON_WA_SIZE]; /* 3 slots */ + #ifdef DBG ULONG g_DebugLevel = DEBUG_LEVEL_MID; #else ULONG g_DebugLevel = DEBUG_LEVEL_LOW; #endif +#pragma data_seg() + + //----------------------------------------------------- #define MAX_HCA_NUM 16 +// flags in below structure +#define FIX_ORDERING_BUG 1 typedef struct mfile_ibal_t { mfile s; ib_al_handle_t h_al; ib_ca_handle_t h_ca; map_crspace cr_map; + + int bugs; + int slot_num; + u_int32_t hermon_wa_slot; /* apply hermon cr write workaround */ + int hermon_wa_last_op_write; + u_int64_t hermon_wa_max_retries; + u_int64_t hermon_wa_num_of_writes; + u_int64_t hermon_wa_num_of_retry_writes; } mfile_ibal; +// Return slot_num or -1 when no free slots +static int __get_slot() +{ + int i, prev_val; + + for (i = 0; i < HERMON_WA_SIZE; ++i) { + prev_val = InterlockedCompareExchange( + (LONG volatile* )&g_Slots[i], 0, 1 ); + if ( prev_val ) + return i; + } + return -1; +} + +static void __put_slot(int slot_num) +{ + int prev_val; + prev_val = InterlockedExchange( + (LONG volatile* )&g_Slots[slot_num], 1 ); + if ( !prev_val ) + DPRINT1(("Wrong semaphore %d value \n", slot_num)); + +} + BOOL APIENTRY DllMain( HANDLE hModule, DWORD ul_reason_for_call, LPVOID lpReserved ) { + int i; char* pszDbgLevel; switch (ul_reason_for_call) { @@ -59,6 +108,9 @@ if (pszDbgLevel) { g_DebugLevel = atol(pszDbgLevel); } + // set slot data + for (i = 0; i < HERMON_WA_SIZE; ++i) + g_Slots[i] = 1; break; @@ -85,13 +137,20 @@ #define SINAI_4X_CONF_DEV_ID 24205 #define SINAI_8X_DEV_ID 25204 #define SINAI_8X_CONF_DEV_ID 25205 +#define HERMON_SDR_DEV_ID 25408 +#define HERMON_DDR_DEV_ID 25418 +#define HERMON_QDR_DEV_ID 25428 +#define HERMON_DDR_PCI5_DEV_ID 26418 +#define HERMON_QDR_PCI5_DEV_ID 26428 +#define HERMON_CONF_DEV_ID 401 #define IS_CONF_DEV(dev_id) \ - ((dev_id == TAVOR_CONF_DEV_ID) || \ - (dev_id == ARBEL_TM_CONF_DEV_ID) || \ - (dev_id == ARBEL_CONF_DEV_ID) || \ - (dev_id == SINAI_4X_CONF_DEV_ID) || \ - (dev_id == SINAI_8X_CONF_DEV_ID)) + ((dev_id == TAVOR_CONF_DEV_ID) || \ + (dev_id == ARBEL_TM_CONF_DEV_ID) || \ + (dev_id == ARBEL_CONF_DEV_ID) || \ + (dev_id == SINAI_4X_CONF_DEV_ID) || \ + (dev_id == SINAI_8X_CONF_DEV_ID) || \ + (dev_id == HERMON_CONF_DEV_ID)) #define MAX_DEV_NAME 32 typedef struct { @@ -115,7 +174,15 @@ { SINAI_4X_DEV_ID, "InfiniHost_III_Lx", MDEVS_TAVOR }, { SINAI_4X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR }, { SINAI_8X_DEV_ID, "InfiniHost_III_Lx", MDEVS_TAVOR }, - { SINAI_8X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR }, + { SINAI_8X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR }, + + { HERMON_SDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_DDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_QDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_DDR_PCI5_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + { HERMON_QDR_PCI5_DEV_ID, "ConnectX", MDEVS_TAVOR_CR }, + + { HERMON_CONF_DEV_ID, "ConnectXBd", MDEVS_TAVOR_CR }, }; #define DEVICE_DB_SIZE (sizeof(db) / sizeof(DEVICE_DB_T)) @@ -123,6 +190,14 @@ char *dsuffix[] = { "conf", "_cr", "_uar", "_ddr"}; #define MASKS_SIZE (sizeof(dmasks) / sizeof(Mdevs)) +static int is_hermon(USHORT dev_id) +{ + return (dev_id == HERMON_SDR_DEV_ID) || + (dev_id == HERMON_DDR_DEV_ID) || + (dev_id == HERMON_QDR_DEV_ID) || + (dev_id == HERMON_DDR_PCI5_DEV_ID) || + (dev_id == HERMON_QDR_PCI5_DEV_ID); +} // Return: < 0 - Error. > 0 - Numbern of characters written (including last '\0') int create_mst_names_by_dev_id(USHORT dev_id, int dev_ix, int mask, char *name, int name_len, int *cnt) @@ -431,8 +506,70 @@ } + // +// Hermon PCI ordering bug work around (in kernel): // +// Toggle bit is done using a HW semaphore. 3 slots (sems) are allocated for MST tools and +// are dynamically allocated by the mst_pci.o module. 1 slot is reserved for external tools usage. +// This means to more than 3 tools that use the PCI dev can run simultaniously. +// +// Open: +// If the wa is enabled ("MTCR_HERMON_WA" env variable does NOT exist or exist and equals 1): +// and HERMON-A0 dev id is read: +// get a vacant slot - ioctl( , PCI_HERMON_WA ,); +// read the sem in the slot (to lock it). +// +// +// Flush (hermon_wa_write_confirm_sem): +// If enabled on first read after write: +// write 0 to sem. +// read sem untlill sem = 0; +// +// Close/release +// Zero the slot. +// + +int hermon_wa_write_confirm_sem(mfile* mf, u_int32_t addr) { + u_int64_t max_retries = 0x1000000; + u_int64_t retries = 0; + mfile_ibal *mfi = (mfile_ibal *)mf; + + mfi->hermon_wa_num_of_writes++; + + // clear sem + *((u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)) = 0; + + while (retries < max_retries) { + u_int32_t slot_value ; + + slot_value = __be32_to_cpu(*((volatile u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot))); + if (slot_value == 0) { + + // Good - The written value was read + if (retries) { + // NOT REALLY NEEDED - just to collect statistics + + // printf("-D- Hermon WA addr %06x took %d retries\n", addr, retries); + mfi->hermon_wa_num_of_retry_writes++; + if (retries > mfi->hermon_wa_max_retries) { + mfi->hermon_wa_max_retries = retries; + } + } + return 4; + } else { + // sem is still locked from previous read - read passed the write + retries++; + } + } + + DPRINT1(("-D- Hermon WA addr %06x failed after %ld retries\n", addr, retries)); + return 0; +} + + +// +// // List devices in their MST compatible names. // Each device type is indexed sepetrately. // @@ -629,7 +766,7 @@ } } else { - int bar_num; + int bar_num, slot_num; // Type of file mf->s.tp = MST_PCI; @@ -649,8 +786,36 @@ } mf->s.ptr = (void*)(ULONG_PTR)mf->cr_map.va; - } - + + // check whether we need to perform the workaround + if (is_hermon(dev_id)) { + char* hermon_wa_env = getenv("MTCR_HERMON_WA"); + // if there is no env variable or it = "0" - we perform the workaround + if (hermon_wa_env == NULL || strcmp(hermon_wa_env, "0")) { + // perform the workaround only for A0 revision + if (access_type == MDEVS_TAVOR_CR) { + u_int32_t rev = 0; + mread4((mfile*)mf, 0xf0014, &rev); + if (rev == 0xa00190) { // TODO: Should I ignore REV here ? + slot_num = __get_slot(); + if (slot_num < 0) // no free slot + goto ErrExit; + mf->bugs |= FIX_ORDERING_BUG; + mf->slot_num = slot_num; + } + } + } + } + + // prepare for the workaround + if (mf->bugs & FIX_ORDERING_BUG) { // Hermon WA setup + DPRINT1(("-D- Hermon WA setup\n")); + mf->hermon_wa_slot = HERMON_WA_BASE + 4*slot_num; + hermon_wa_write_confirm_sem((mfile*)mf, mf->hermon_wa_slot); + } + + } + } else if (dev_id == DEVASYS_DEV_ID) { // Type of file h = usb_open(); @@ -702,6 +867,16 @@ if (mf->tp == MST_PCICONF) { ibal_access(mfi->h_ca, 0x0, &stub, 4, FW_CLOSE_IF ); } else if (mf->tp = MST_PCI) { + + if (mfi->bugs & FIX_ORDERING_BUG) { + __put_slot( mfi->slot_num ); + DPRINT1(("-D- Hermon WA stats:\n")); + DPRINT1(("-D- : num of write flushes: %8ld\n", mfi->hermon_wa_num_of_writes)); + DPRINT1(("-D- : num of retry flushes: %8ld\n", mfi->hermon_wa_num_of_retry_writes)); + DPRINT1(("-D- : max_retries: %8ld\n", mfi->hermon_wa_max_retries)); + } + + if (mfi->cr_map.size) { if (ibal_access(mfi->h_ca, 0x0, NULL, 0, FW_UNMAP_CRSPACE ) != IB_SUCCESS) {
_______________________________________________ ofw mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
