256-bit invaildation queue descriptor could be used for both abort DMA mode and legacy mode.
Signed-off-by: Sheng Wei <w.sh...@intel.com> Cc: Ray Ni <ray...@intel.com> Cc: Rangasai V Chaganty <rangasai.v.chaga...@intel.com> Cc: Jenny Huang <jenny.hu...@intel.com> Cc: Robert Kowalewski <robert.kowalew...@intel.com> --- .../VTd/IntelVTdDmarPei/IntelVTdDmar.c | 179 ++++++++----- .../VTd/IntelVTdDmarPei/IntelVTdDmarPei.h | 12 +- .../Feature/VTd/IntelVTdDxe/DmaProtection.c | 3 + .../Feature/VTd/IntelVTdDxe/DmaProtection.h | 12 +- .../Feature/VTd/IntelVTdDxe/VtdReg.c | 235 +++++++++++------- .../Include/IndustryStandard/Vtd.h | 65 ++++- 6 files changed, 347 insertions(+), 159 deletions(-) diff --git a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmar.c b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmar.c index af85a3d8e..ae9135010 100644 --- a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmar.c +++ b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmar.c @@ -79,10 +79,9 @@ PerpareCacheInvalidationInterface ( IN VTD_UNIT_INFO *VTdUnitInfo ) { - UINT16 QueueSize; - UINT64 Reg64; UINT32 Reg32; VTD_ECAP_REG ECapReg; + VTD_IQA_REG IqaReg; UINTN VtdUnitBaseAddress; VtdUnitBaseAddress = VTdUnitInfo->VtdUnitBaseAddress; @@ -121,20 +120,25 @@ PerpareCacheInvalidationInterface ( // // Setup the IQ address, size and descriptor width through the Invalidation Queue Address Register // - if (VTdUnitInfo->QiDesc == NULL) { - QueueSize = 0; - VTdUnitInfo->QiDescLength = 1 << (QueueSize + 8); - VTdUnitInfo->QiDesc = (QI_DESC *) AllocatePages (EFI_SIZE_TO_PAGES (sizeof (QI_DESC) * VTdUnitInfo->QiDescLength)); - if (VTdUnitInfo->QiDesc == NULL) { + if (VTdUnitInfo->QiDescBuffer == NULL) { + VTdUnitInfo->QiDescBufferSize = (sizeof (QI_256_DESC) * ((UINTN) 1 << (VTD_INVALIDATION_QUEUE_SIZE + 7))); + VTdUnitInfo->QiDescBuffer = AllocatePages (EFI_SIZE_TO_PAGES (VTdUnitInfo->QiDescBufferSize)); + if (VTdUnitInfo->QiDescBuffer == NULL) { DEBUG ((DEBUG_ERROR,"Could not Alloc Invalidation Queue Buffer.\n")); return EFI_OUT_OF_RESOURCES; } } - DEBUG ((DEBUG_INFO, "Invalidation Queue Length : %d\n", VTdUnitInfo->QiDescLength)); - Reg64 = (UINT64) (UINTN) VTdUnitInfo->QiDesc; - Reg64 |= QueueSize; - MmioWrite64 (VtdUnitBaseAddress + R_IQA_REG, Reg64); + DEBUG ((DEBUG_INFO, "Invalidation Queue Buffer Size : %d\n", VTdUnitInfo->QiDescBufferSize)); + // + // 4KB Aligned address + // + IqaReg.Uint64 = (UINT64) (UINTN) VTdUnitInfo->QiDescBuffer; + IqaReg.Bits.DW = VTD_QUEUED_INVALIDATION_DESCRIPTOR_WIDTH; + IqaReg.Bits.QS = VTD_INVALIDATION_QUEUE_SIZE; + MmioWrite64 (VtdUnitBaseAddress + R_IQA_REG, IqaReg.Uint64); + IqaReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQA_REG); + DEBUG ((DEBUG_INFO, "IQA_REG = 0x%lx, IQH_REG = 0x%lx\n", IqaReg.Uint64, MmioRead64 (VtdUnitBaseAddress + R_IQH_REG))); // // Enable the queued invalidation interface through the Global Command Register. @@ -148,8 +152,6 @@ PerpareCacheInvalidationInterface ( Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & B_GSTS_REG_QIES) == 0); - VTdUnitInfo->QiFreeHead = 0; - return EFI_SUCCESS; } @@ -174,10 +176,10 @@ DisableQueuedInvalidationInterface ( Reg32 = MmioRead32 (VTdUnitInfo->VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & B_GSTS_REG_QIES) != 0); - if (VTdUnitInfo->QiDesc != NULL) { - FreePages(VTdUnitInfo->QiDesc, EFI_SIZE_TO_PAGES (sizeof (QI_DESC) * VTdUnitInfo->QiDescLength)); - VTdUnitInfo->QiDesc = NULL; - VTdUnitInfo->QiDescLength = 0; + if (VTdUnitInfo->QiDescBuffer != NULL) { + FreePages(VTdUnitInfo->QiDescBuffer, EFI_SIZE_TO_PAGES (VTdUnitInfo->QiDescBufferSize)); + VTdUnitInfo->QiDescBuffer = NULL; + VTdUnitInfo->QiDescBufferSize = 0; } VTdUnitInfo->EnableQueuedInvalidation = 0; @@ -197,12 +199,15 @@ QueuedInvalidationCheckFault ( IN VTD_UNIT_INFO *VTdUnitInfo ) { - UINT32 FaultReg; + UINT32 FaultReg; + VTD_IQERCD_REG IqercdReg; FaultReg = MmioRead32 (VTdUnitInfo->VtdUnitBaseAddress + R_FSTS_REG); if (FaultReg & (B_FSTS_REG_IQE | B_FSTS_REG_ITE | B_FSTS_REG_ICE)) { - DEBUG((DEBUG_ERROR, "Detect Queue Invalidation Error [0x%08x]\n", FaultReg)); - FaultReg |= (B_FSTS_REG_IQE | B_FSTS_REG_ITE | B_FSTS_REG_ICE); + IqercdReg.Uint64 = MmioRead64 (VTdUnitInfo->VtdUnitBaseAddress + R_IQERCD_REG); + + DEBUG((DEBUG_ERROR, "Detect Queue Invalidation Error [0x%08x] - IQERCD [0x%016lx]\n", FaultReg, IqercdReg.Uint64)); + MmioWrite32 (VTdUnitInfo->VtdUnitBaseAddress + R_FSTS_REG, FaultReg); return RETURN_DEVICE_ERROR; } @@ -223,37 +228,83 @@ QueuedInvalidationCheckFault ( **/ EFI_STATUS SubmitQueuedInvalidationDescriptor ( - IN VTD_UNIT_INFO *VTdUnitInfo, - IN QI_DESC *Desc + IN VTD_UNIT_INFO *VTdUnitInfo, + IN QI_256_DESC *Desc ) { - EFI_STATUS Status; - UINT16 QiDescLength; - QI_DESC *BaseDesc; - UINT64 Reg64Iqt; - UINT64 Reg64Iqh; + EFI_STATUS Status; + UINTN VtdUnitBaseAddress; + UINTN QueueSize; + UINTN QueueTail; + UINTN QueueHead; + QI_DESC *Qi128Desc; + QI_256_DESC *Qi256Desc; + VTD_IQA_REG IqaReg; + VTD_IQT_REG IqtReg; + VTD_IQH_REG IqhReg; if (Desc == NULL) { return EFI_INVALID_PARAMETER; } - QiDescLength = VTdUnitInfo->QiDescLength; - BaseDesc = VTdUnitInfo->QiDesc; - - DEBUG((DEBUG_INFO, "[0x%x] Submit QI Descriptor [0x%016lx, 0x%016lx]\n", VTdUnitInfo->VtdUnitBaseAddress, Desc->Low, Desc->High)); - - BaseDesc[VTdUnitInfo->QiFreeHead].Low = Desc->Low; - BaseDesc[VTdUnitInfo->QiFreeHead].High = Desc->High; - FlushPageTableMemory(VTdUnitInfo, (UINTN) &BaseDesc[VTdUnitInfo->QiFreeHead], sizeof(QI_DESC)); + VtdUnitBaseAddress = VTdUnitInfo->VtdUnitBaseAddress; + IqaReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQA_REG); + if (IqaReg.Bits.IQA == 0) { + DEBUG ((DEBUG_ERROR,"Invalidation Queue Buffer not ready [0x%lx]\n", IqaReg.Uint64)); + return EFI_NOT_READY; + } + IqtReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQT_REG); - DEBUG((DEBUG_INFO,"QI Free Head=0x%x\n", VTdUnitInfo->QiFreeHead)); - VTdUnitInfo->QiFreeHead = (VTdUnitInfo->QiFreeHead + 1) % QiDescLength; + if (IqaReg.Bits.DW == 0) { + // + // 128-bit descriptor + // + QueueSize = (UINTN) (1 << (IqaReg.Bits.QS + 8)); + Qi128Desc = (QI_DESC *) (UINTN) (IqaReg.Bits.IQA << VTD_PAGE_SHIFT); + QueueTail = (UINTN) IqtReg.Bits128Desc.QT; + Qi128Desc += QueueTail; + Qi128Desc->Low = Desc->Uint64[0]; + Qi128Desc->High = Desc->Uint64[1]; + FlushPageTableMemory (VTdUnitInfo, (UINTN) Qi128Desc, sizeof(QI_DESC)); + QueueTail = (QueueTail + 1) % QueueSize; + + DEBUG ((DEBUG_VERBOSE, "[0x%x] Submit QI Descriptor 0x%x [0x%016lx, 0x%016lx]\n", + VtdUnitBaseAddress, + QueueTail, + Desc->Uint64[0], + Desc->Uint64[1])); + + IqtReg.Bits128Desc.QT = QueueTail; + } else { + // + // 256-bit descriptor + // + QueueSize = (UINTN) (1 << (IqaReg.Bits.QS + 7)); + Qi256Desc = (QI_256_DESC *) (UINTN) (IqaReg.Bits.IQA << VTD_PAGE_SHIFT); + QueueTail = (UINTN) IqtReg.Bits256Desc.QT; + Qi256Desc += QueueTail; + Qi256Desc->Uint64[0] = Desc->Uint64[0]; + Qi256Desc->Uint64[1] = Desc->Uint64[1]; + Qi256Desc->Uint64[2] = Desc->Uint64[2]; + Qi256Desc->Uint64[3] = Desc->Uint64[3]; + FlushPageTableMemory (VTdUnitInfo, (UINTN) Qi256Desc, sizeof(QI_256_DESC)); + QueueTail = (QueueTail + 1) % QueueSize; + + DEBUG ((DEBUG_VERBOSE, "[0x%x] Submit QI Descriptor 0x%x [0x%016lx, 0x%016lx, 0x%016lx, 0x%016lx]\n", + VtdUnitBaseAddress, + QueueTail, + Desc->Uint64[0], + Desc->Uint64[1], + Desc->Uint64[2], + Desc->Uint64[3])); + + IqtReg.Bits256Desc.QT = QueueTail; + } // // Update the HW tail register indicating the presence of new descriptors. // - Reg64Iqt = VTdUnitInfo->QiFreeHead << DMAR_IQ_SHIFT; - MmioWrite64 (VTdUnitInfo->VtdUnitBaseAddress + R_IQT_REG, Reg64Iqt); + MmioWrite64 (VtdUnitBaseAddress + R_IQT_REG, IqtReg.Uint64); Status = EFI_SUCCESS; do { @@ -263,10 +314,15 @@ SubmitQueuedInvalidationDescriptor ( break; } - Reg64Iqh = MmioRead64 (VTdUnitInfo->VtdUnitBaseAddress + R_IQH_REG); - } while (Reg64Iqt != Reg64Iqh); + IqhReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQH_REG); + if (IqaReg.Bits.DW == 0) { + QueueHead = (UINTN) IqhReg.Bits128Desc.QH; + } else { + QueueHead = (UINTN) IqhReg.Bits256Desc.QH; + } + } while (QueueTail != QueueHead); - DEBUG((DEBUG_ERROR,"SubmitQueuedInvalidationDescriptor end\n")); + DEBUG((DEBUG_INFO,"SubmitQueuedInvalidationDescriptor end\n")); return Status; } @@ -281,7 +337,7 @@ InvalidateContextCache ( ) { UINT64 Reg64; - QI_DESC QiDesc; + QI_256_DESC QiDesc; if (VTdUnitInfo->EnableQueuedInvalidation == 0) { // @@ -304,8 +360,10 @@ InvalidateContextCache ( // // Queued Invalidation // - QiDesc.Low = QI_CC_FM(0) | QI_CC_SID(0) | QI_CC_DID(0) | QI_CC_GRAN(1) | QI_CC_TYPE; - QiDesc.High = 0; + QiDesc.Uint64[0] = QI_CC_FM(0) | QI_CC_SID(0) | QI_CC_DID(0) | QI_CC_GRAN(1) | QI_CC_TYPE; + QiDesc.Uint64[1] = 0; + QiDesc.Uint64[2] = 0; + QiDesc.Uint64[3] = 0; return SubmitQueuedInvalidationDescriptor(VTdUnitInfo, &QiDesc); } @@ -326,7 +384,7 @@ InvalidateIOTLB ( UINT64 Reg64; VTD_ECAP_REG ECapReg; VTD_CAP_REG CapReg; - QI_DESC QiDesc; + QI_256_DESC QiDesc; if (VTdUnitInfo->EnableQueuedInvalidation == 0) { // @@ -352,8 +410,10 @@ InvalidateIOTLB ( // Queued Invalidation // CapReg.Uint64 = MmioRead64 (VTdUnitInfo->VtdUnitBaseAddress + R_CAP_REG); - QiDesc.Low = QI_IOTLB_DID(0) | QI_IOTLB_DR(CAP_READ_DRAIN(CapReg.Uint64)) | QI_IOTLB_DW(CAP_WRITE_DRAIN(CapReg.Uint64)) | QI_IOTLB_GRAN(1) | QI_IOTLB_TYPE; - QiDesc.High = QI_IOTLB_ADDR(0) | QI_IOTLB_IH(0) | QI_IOTLB_AM(0); + QiDesc.Uint64[0] = QI_IOTLB_DID(0) | QI_IOTLB_DR(CAP_READ_DRAIN(CapReg.Uint64)) | QI_IOTLB_DW(CAP_WRITE_DRAIN(CapReg.Uint64)) | QI_IOTLB_GRAN(1) | QI_IOTLB_TYPE; + QiDesc.Uint64[1] = QI_IOTLB_ADDR(0) | QI_IOTLB_IH(0) | QI_IOTLB_AM(0); + QiDesc.Uint64[2] = 0; + QiDesc.Uint64[3] = 0; return SubmitQueuedInvalidationDescriptor(VTdUnitInfo, &QiDesc); } @@ -390,6 +450,7 @@ ClearGlobalCommandRegisterBits ( do { Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & BitMask) == BitMask); + DEBUG ((DEBUG_INFO, "GSTS_REG : 0x%08x \n", Reg32)); } /** @@ -421,6 +482,7 @@ SetGlobalCommandRegisterBits ( do { Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & BitMask) == 0); + DEBUG ((DEBUG_INFO, "GSTS_REG : 0x%08x \n", Reg32)); } /** @@ -451,11 +513,6 @@ EnableDmarPreMem ( Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); DEBUG ((DEBUG_INFO, "EnableDmarPreMem: R_GSTS_REG = 0x%x \n", Reg32)); - // - // Init DMAr Fault Event and Data registers - // - Reg32 = MmioRead32 (VtdUnitBaseAddress + R_FEDATA_REG); - // // Write Buffer Flush // @@ -513,6 +570,9 @@ EnableDmar ( // MmioWrite64 (VtdUnitBaseAddress + R_RTADDR_REG, (UINT64) (RootEntryTable | V_RTADDR_REG_TTM_ADM)); + DEBUG ((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); + SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); + DEBUG((DEBUG_INFO, "Enable Abort DMA Mode...\n")); SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_TE); @@ -520,16 +580,10 @@ EnableDmar ( DEBUG ((DEBUG_INFO, "RootEntryTable 0x%x \n", RootEntryTable)); MmioWrite64 (VtdUnitBaseAddress + R_RTADDR_REG, (UINT64) RootEntryTable); + DEBUG ((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); + SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); } - DEBUG ((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); - SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); - - // - // Init DMAr Fault Event and Data registers - // - MmioRead32 (VtdUnitBaseAddress + R_FEDATA_REG); - // // Write Buffer Flush before invalidation // @@ -552,6 +606,9 @@ EnableDmar ( DEBUG ((DEBUG_INFO, "RootEntryTable 0x%x \n", RootEntryTable)); MmioWrite64 (VtdUnitBaseAddress + R_RTADDR_REG, (UINT64) RootEntryTable); + + DEBUG ((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); + SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); } // diff --git a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmarPei.h b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmarPei.h index 5ade9ec35..db860bf80 100644 --- a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmarPei.h +++ b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDmarPei/IntelVTdDmarPei.h @@ -11,6 +11,13 @@ #define VTD_64BITS_ADDRESS(Lo, Hi) (LShiftU64 (Lo, 12) | LShiftU64 (Hi, 32)) +// +// Use 256-bit descriptor +// Queue size is 128. +// +#define VTD_QUEUED_INVALIDATION_DESCRIPTOR_WIDTH 1 +#define VTD_INVALIDATION_QUEUE_SIZE 0 + typedef struct { BOOLEAN Done; UINTN VtdUnitBaseAddress; @@ -21,9 +28,8 @@ typedef struct { VTD_ECAP_REG ECapReg; BOOLEAN Is5LevelPaging; UINT8 EnableQueuedInvalidation; - UINT16 QiDescLength; - QI_DESC *QiDesc; - UINT16 QiFreeHead; + VOID *QiDescBuffer; + UINTN QiDescBufferSize; UINTN FixedSecondLevelPagingEntry; UINTN RootEntryTable; UINTN ExtRootEntryTable; diff --git a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.c b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.c index 628565ee7..878c952b7 100644 --- a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.c +++ b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.c @@ -496,6 +496,9 @@ SetupVtd ( if (EFI_ERROR (Status)) { return; } + + DumpVtdIfError (); + DEBUG ((DEBUG_INFO, "PrepareVtdConfig\n")); PrepareVtdConfig (); diff --git a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.h b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.h index 7dd29a243..b878a0736 100644 --- a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.h +++ b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/DmaProtection.h @@ -46,6 +46,13 @@ #define VTD_TPL_LEVEL TPL_NOTIFY +// +// Use 256-bit descriptor +// Queue size is 128. +// +#define VTD_QUEUED_INVALIDATION_DESCRIPTOR_WIDTH 1 +#define VTD_INVALIDATION_QUEUE_SIZE 0 + // // This is the initial max PCI DATA number. // The number may be enlarged later. @@ -81,9 +88,8 @@ typedef struct { PCI_DEVICE_INFORMATION PciDeviceInfo; BOOLEAN Is5LevelPaging; UINT8 EnableQueuedInvalidation; - UINT16 QiDescLength; - QI_DESC *QiDesc; - UINT16 QiFreeHead; + VOID *QiDescBuffer; + UINTN QiDescBufferSize; } VTD_UNIT_INFORMATION; // diff --git a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/VtdReg.c b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/VtdReg.c index 8e834f4c4..30dab4a64 100644 --- a/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/VtdReg.c +++ b/Silicon/Intel/IntelSiliconPkg/Feature/VTd/IntelVTdDxe/VtdReg.c @@ -68,71 +68,78 @@ PerpareCacheInvalidationInterface ( IN UINTN VtdIndex ) { - UINT16 QueueSize; - UINT64 Reg64; - UINT32 Reg32; + UINT32 Reg32; + VTD_IQA_REG IqaReg; + VTD_UNIT_INFORMATION *VTdUnitInfo; + UINTN VtdUnitBaseAddress; - if (mVtdUnitInformation[VtdIndex].VerReg.Bits.Major <= 5) { - mVtdUnitInformation[VtdIndex].EnableQueuedInvalidation = 0; + VTdUnitInfo = &mVtdUnitInformation[VtdIndex]; + VtdUnitBaseAddress = VTdUnitInfo->VtdUnitBaseAddress; + + if (VTdUnitInfo->VerReg.Bits.Major <= 5) { + VTdUnitInfo->EnableQueuedInvalidation = 0; DEBUG ((DEBUG_INFO, "Use Register-based Invalidation Interface for engine [%d]\n", VtdIndex)); return EFI_SUCCESS; } - if (mVtdUnitInformation[VtdIndex].ECapReg.Bits.QI == 0) { + if (VTdUnitInfo->ECapReg.Bits.QI == 0) { DEBUG ((DEBUG_ERROR, "Hardware does not support queued invalidations interface for engine [%d]\n", VtdIndex)); return EFI_UNSUPPORTED; } - mVtdUnitInformation[VtdIndex].EnableQueuedInvalidation = 1; + VTdUnitInfo->EnableQueuedInvalidation = 1; DEBUG ((DEBUG_INFO, "Use Queued Invalidation Interface for engine [%d]\n", VtdIndex)); - Reg32 = MmioRead32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GSTS_REG); + Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); if ((Reg32 & B_GSTS_REG_QIES) != 0) { DEBUG ((DEBUG_ERROR,"Queued Invalidation Interface was enabled.\n")); Reg32 &= (~B_GSTS_REG_QIES); - MmioWrite32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GCMD_REG, Reg32); + MmioWrite32 (VtdUnitBaseAddress + R_GCMD_REG, Reg32); do { - Reg32 = MmioRead32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GSTS_REG); + Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & B_GSTS_REG_QIES) != 0); } // // Initialize the Invalidation Queue Tail Register to zero. // - MmioWrite64 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_IQT_REG, 0); + MmioWrite64 (VtdUnitBaseAddress + R_IQT_REG, 0); // // Setup the IQ address, size and descriptor width through the Invalidation Queue Address Register // - QueueSize = 0; - mVtdUnitInformation[VtdIndex].QiDescLength = 1 << (QueueSize + 8); - mVtdUnitInformation[VtdIndex].QiDesc = (QI_DESC *) AllocatePages (EFI_SIZE_TO_PAGES(sizeof(QI_DESC) * mVtdUnitInformation[VtdIndex].QiDescLength)); - - if (mVtdUnitInformation[VtdIndex].QiDesc == NULL) { - mVtdUnitInformation[VtdIndex].QiDescLength = 0; - DEBUG ((DEBUG_ERROR,"Could not Alloc Invalidation Queue Buffer.\n")); - return EFI_OUT_OF_RESOURCES; + if (VTdUnitInfo->QiDescBuffer == NULL) { + VTdUnitInfo->QiDescBufferSize = (sizeof (QI_256_DESC) * ((UINTN) 1 << (VTD_INVALIDATION_QUEUE_SIZE + 7))); + VTdUnitInfo->QiDescBuffer = AllocatePages (EFI_SIZE_TO_PAGES (VTdUnitInfo->QiDescBufferSize)); + if (VTdUnitInfo->QiDescBuffer == NULL) { + DEBUG ((DEBUG_ERROR,"Could not Alloc Invalidation Queue Buffer.\n")); + return EFI_OUT_OF_RESOURCES; + } } - DEBUG ((DEBUG_INFO, "Invalidation Queue Length : %d\n", mVtdUnitInformation[VtdIndex].QiDescLength)); - Reg64 = (UINT64)(UINTN)mVtdUnitInformation[VtdIndex].QiDesc; - Reg64 |= QueueSize; - MmioWrite64 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_IQA_REG, Reg64); + DEBUG ((DEBUG_INFO, "Invalidation Queue Buffer Size : %d\n", VTdUnitInfo->QiDescBufferSize)); + // + // 4KB Aligned address + // + IqaReg.Uint64 = (UINT64) (UINTN) VTdUnitInfo->QiDescBuffer; + IqaReg.Bits.DW = VTD_QUEUED_INVALIDATION_DESCRIPTOR_WIDTH; + IqaReg.Bits.QS = VTD_INVALIDATION_QUEUE_SIZE; + MmioWrite64 (VtdUnitBaseAddress + R_IQA_REG, IqaReg.Uint64); + IqaReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQA_REG); + DEBUG ((DEBUG_INFO, "IQA_REG = 0x%lx, IQH_REG = 0x%lx\n", IqaReg.Uint64, MmioRead64 (VtdUnitBaseAddress + R_IQH_REG))); // // Enable the queued invalidation interface through the Global Command Register. // When enabled, hardware sets the QIES field in the Global Status Register. // - Reg32 = MmioRead32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GSTS_REG); + Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); Reg32 |= B_GMCD_REG_QIE; - MmioWrite32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GCMD_REG, Reg32); + MmioWrite32 (VtdUnitBaseAddress + R_GCMD_REG, Reg32); DEBUG ((DEBUG_INFO, "Enable Queued Invalidation Interface. GCMD_REG = 0x%x\n", Reg32)); do { - Reg32 = MmioRead32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GSTS_REG); + Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & B_GSTS_REG_QIES) == 0); - mVtdUnitInformation[VtdIndex].QiFreeHead = 0; - return EFI_SUCCESS; } @@ -146,21 +153,24 @@ DisableQueuedInvalidationInterface ( IN UINTN VtdIndex ) { - UINT32 Reg32; + UINT32 Reg32; + VTD_UNIT_INFORMATION *VTdUnitInfo; - if (mVtdUnitInformation[VtdIndex].EnableQueuedInvalidation != 0) { - Reg32 = MmioRead32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GSTS_REG); + VTdUnitInfo = &mVtdUnitInformation[VtdIndex]; + + if (VTdUnitInfo->EnableQueuedInvalidation != 0) { + Reg32 = MmioRead32 (VTdUnitInfo->VtdUnitBaseAddress + R_GSTS_REG); Reg32 &= (~B_GMCD_REG_QIE); - MmioWrite32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GCMD_REG, Reg32); + MmioWrite32 (VTdUnitInfo->VtdUnitBaseAddress + R_GCMD_REG, Reg32); DEBUG ((DEBUG_INFO, "Disable Queued Invalidation Interface. GCMD_REG = 0x%x\n", Reg32)); do { - Reg32 = MmioRead32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_GSTS_REG); + Reg32 = MmioRead32 (VTdUnitInfo->VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & B_GSTS_REG_QIES) != 0); - if (mVtdUnitInformation[VtdIndex].QiDesc != NULL) { - FreePages(mVtdUnitInformation[VtdIndex].QiDesc, EFI_SIZE_TO_PAGES(sizeof(QI_DESC) * mVtdUnitInformation[VtdIndex].QiDescLength)); - mVtdUnitInformation[VtdIndex].QiDesc = NULL; - mVtdUnitInformation[VtdIndex].QiDescLength = 0; + if (VTdUnitInfo->QiDescBuffer != NULL) { + FreePages(VTdUnitInfo->QiDescBuffer, EFI_SIZE_TO_PAGES (VTdUnitInfo->QiDescBufferSize)); + VTdUnitInfo->QiDescBuffer = NULL; + VTdUnitInfo->QiDescBufferSize = 0; } mVtdUnitInformation[VtdIndex].EnableQueuedInvalidation = 0; @@ -180,27 +190,16 @@ QueuedInvalidationCheckFault ( IN UINTN VtdIndex ) { - UINT32 FaultReg; + UINT32 FaultReg; + VTD_IQERCD_REG IqercdReg; FaultReg = MmioRead32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_FSTS_REG); - if (FaultReg & B_FSTS_REG_IQE) { - DEBUG((DEBUG_ERROR, "Detect Invalidation Queue Error [0x%08x]\n", FaultReg)); - FaultReg |= B_FSTS_REG_IQE; - MmioWrite32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_FSTS_REG, FaultReg); - return RETURN_DEVICE_ERROR; - } + if (FaultReg & (B_FSTS_REG_IQE | B_FSTS_REG_ITE | B_FSTS_REG_ICE)) { + IqercdReg.Uint64 = MmioRead64 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_IQERCD_REG); - if (FaultReg & B_FSTS_REG_ITE) { - DEBUG((DEBUG_ERROR, "Detect Invalidation Time-out Error [0x%08x]\n", FaultReg)); - FaultReg |= B_FSTS_REG_ITE; - MmioWrite32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_FSTS_REG, FaultReg); - return RETURN_DEVICE_ERROR; - } + DEBUG((DEBUG_ERROR, "Detect Queue Invalidation Error [0x%08x] - IQERCD [0x%016lx]\n", FaultReg, IqercdReg.Uint64)); - if (FaultReg & B_FSTS_REG_ICE) { - DEBUG((DEBUG_ERROR, "Detect Invalidation Completion Error [0x%08x]\n", FaultReg)); - FaultReg |= B_FSTS_REG_ICE; MmioWrite32 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_FSTS_REG, FaultReg); return RETURN_DEVICE_ERROR; } @@ -221,36 +220,83 @@ QueuedInvalidationCheckFault ( **/ EFI_STATUS SubmitQueuedInvalidationDescriptor ( - IN UINTN VtdIndex, - IN QI_DESC *Desc + IN UINTN VtdIndex, + IN QI_256_DESC *Desc ) { - EFI_STATUS Status; - UINT16 QiDescLength; - QI_DESC *BaseDesc; - UINT64 Reg64Iqt; - UINT64 Reg64Iqh; + EFI_STATUS Status; + UINTN VtdUnitBaseAddress; + UINTN QueueSize; + UINTN QueueTail; + UINTN QueueHead; + QI_DESC *Qi128Desc; + QI_256_DESC *Qi256Desc; + VTD_IQA_REG IqaReg; + VTD_IQT_REG IqtReg; + VTD_IQH_REG IqhReg; if (Desc == NULL) { return EFI_INVALID_PARAMETER; } - QiDescLength = mVtdUnitInformation[VtdIndex].QiDescLength; - BaseDesc = mVtdUnitInformation[VtdIndex].QiDesc; - - DEBUG((DEBUG_VERBOSE, "[%d] Submit QI Descriptor [0x%08x, 0x%08x] Free Head (%d)\n", VtdIndex, Desc->Low, Desc->High, mVtdUnitInformation[VtdIndex].QiFreeHead)); - - BaseDesc[mVtdUnitInformation[VtdIndex].QiFreeHead].Low = Desc->Low; - BaseDesc[mVtdUnitInformation[VtdIndex].QiFreeHead].High = Desc->High; - FlushPageTableMemory(VtdIndex, (UINTN) &BaseDesc[mVtdUnitInformation[VtdIndex].QiFreeHead], sizeof(QI_DESC)); + VtdUnitBaseAddress = mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress; + IqaReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQA_REG); + if (IqaReg.Bits.IQA == 0) { + DEBUG ((DEBUG_ERROR,"Invalidation Queue Buffer not ready [0x%lx]\n", IqaReg.Uint64)); + return EFI_NOT_READY; + } + IqtReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQT_REG); - mVtdUnitInformation[VtdIndex].QiFreeHead = (mVtdUnitInformation[VtdIndex].QiFreeHead + 1) % QiDescLength; + if (IqaReg.Bits.DW == 0) { + // + // 128-bit descriptor + // + QueueSize = (UINTN) (1 << (IqaReg.Bits.QS + 8)); + Qi128Desc = (QI_DESC *) (UINTN) (IqaReg.Bits.IQA << VTD_PAGE_SHIFT); + QueueTail = (UINTN) IqtReg.Bits128Desc.QT; + Qi128Desc += QueueTail; + Qi128Desc->Low = Desc->Uint64[0]; + Qi128Desc->High = Desc->Uint64[1]; + FlushPageTableMemory (VtdIndex, (UINTN) Qi128Desc, sizeof(QI_DESC)); + QueueTail = (QueueTail + 1) % QueueSize; + + DEBUG ((DEBUG_VERBOSE, "[0x%x] Submit QI Descriptor 0x%x [0x%016lx, 0x%016lx]\n", + VtdUnitBaseAddress, + QueueTail, + Desc->Uint64[0], + Desc->Uint64[1])); + + IqtReg.Bits128Desc.QT = QueueTail; + } else { + // + // 256-bit descriptor + // + QueueSize = (UINTN) (1 << (IqaReg.Bits.QS + 7)); + Qi256Desc = (QI_256_DESC *) (UINTN) (IqaReg.Bits.IQA << VTD_PAGE_SHIFT); + QueueTail = (UINTN) IqtReg.Bits256Desc.QT; + Qi256Desc += QueueTail; + Qi256Desc->Uint64[0] = Desc->Uint64[0]; + Qi256Desc->Uint64[1] = Desc->Uint64[1]; + Qi256Desc->Uint64[2] = Desc->Uint64[2]; + Qi256Desc->Uint64[3] = Desc->Uint64[3]; + FlushPageTableMemory (VtdIndex, (UINTN) Qi256Desc, sizeof(QI_256_DESC)); + QueueTail = (QueueTail + 1) % QueueSize; + + DEBUG ((DEBUG_VERBOSE, "[0x%x] Submit QI Descriptor 0x%x [0x%016lx, 0x%016lx, 0x%016lx, 0x%016lx]\n", + VtdUnitBaseAddress, + QueueTail, + Desc->Uint64[0], + Desc->Uint64[1], + Desc->Uint64[2], + Desc->Uint64[3])); + + IqtReg.Bits256Desc.QT = QueueTail; + } // // Update the HW tail register indicating the presence of new descriptors. // - Reg64Iqt = mVtdUnitInformation[VtdIndex].QiFreeHead << DMAR_IQ_SHIFT; - MmioWrite64 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_IQT_REG, Reg64Iqt); + MmioWrite64 (VtdUnitBaseAddress + R_IQT_REG, IqtReg.Uint64); Status = EFI_SUCCESS; do { @@ -260,8 +306,13 @@ SubmitQueuedInvalidationDescriptor ( break; } - Reg64Iqh = MmioRead64 (mVtdUnitInformation[VtdIndex].VtdUnitBaseAddress + R_IQH_REG); - } while (Reg64Iqt != Reg64Iqh); + IqhReg.Uint64 = MmioRead64 (VtdUnitBaseAddress + R_IQH_REG); + if (IqaReg.Bits.DW == 0) { + QueueHead = (UINTN) IqhReg.Bits128Desc.QH; + } else { + QueueHead = (UINTN) IqhReg.Bits256Desc.QH; + } + } while (QueueTail != QueueHead); return Status; } @@ -276,8 +327,8 @@ InvalidateContextCache ( IN UINTN VtdIndex ) { - UINT64 Reg64; - QI_DESC QiDesc; + UINT64 Reg64; + QI_256_DESC QiDesc; if (mVtdUnitInformation[VtdIndex].EnableQueuedInvalidation == 0) { // @@ -300,8 +351,10 @@ InvalidateContextCache ( // // Queued Invalidation // - QiDesc.Low = QI_CC_FM(0) | QI_CC_SID(0) | QI_CC_DID(0) | QI_CC_GRAN(1) | QI_CC_TYPE; - QiDesc.High = 0; + QiDesc.Uint64[0] = QI_CC_FM(0) | QI_CC_SID(0) | QI_CC_DID(0) | QI_CC_GRAN(1) | QI_CC_TYPE; + QiDesc.Uint64[1] = 0; + QiDesc.Uint64[2] = 0; + QiDesc.Uint64[3] = 0; return SubmitQueuedInvalidationDescriptor(VtdIndex, &QiDesc); } @@ -318,8 +371,8 @@ InvalidateIOTLB ( IN UINTN VtdIndex ) { - UINT64 Reg64; - QI_DESC QiDesc; + UINT64 Reg64; + QI_256_DESC QiDesc; if (mVtdUnitInformation[VtdIndex].EnableQueuedInvalidation == 0) { // @@ -342,8 +395,10 @@ InvalidateIOTLB ( // // Queued Invalidation // - QiDesc.Low = QI_IOTLB_DID(0) | QI_IOTLB_DR(CAP_READ_DRAIN(mVtdUnitInformation[VtdIndex].CapReg.Uint64)) | QI_IOTLB_DW(CAP_WRITE_DRAIN(mVtdUnitInformation[VtdIndex].CapReg.Uint64)) | QI_IOTLB_GRAN(1) | QI_IOTLB_TYPE; - QiDesc.High = QI_IOTLB_ADDR(0) | QI_IOTLB_IH(0) | QI_IOTLB_AM(0); + QiDesc.Uint64[0] = QI_IOTLB_DID(0) | QI_IOTLB_DR(CAP_READ_DRAIN(mVtdUnitInformation[VtdIndex].CapReg.Uint64)) | QI_IOTLB_DW(CAP_WRITE_DRAIN(mVtdUnitInformation[VtdIndex].CapReg.Uint64)) | QI_IOTLB_GRAN(1) | QI_IOTLB_TYPE; + QiDesc.Uint64[1] = QI_IOTLB_ADDR(0) | QI_IOTLB_IH(0) | QI_IOTLB_AM(0); + QiDesc.Uint64[2] = 0; + QiDesc.Uint64[3] = 0; return SubmitQueuedInvalidationDescriptor(VtdIndex, &QiDesc); } @@ -504,6 +559,7 @@ ClearGlobalCommandRegisterBits ( do { Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & BitMask) == BitMask); + DEBUG ((DEBUG_INFO, "GSTS_REG : 0x%08x \n", Reg32)); } /** @@ -535,6 +591,7 @@ SetGlobalCommandRegisterBits ( do { Reg32 = MmioRead32 (VtdUnitBaseAddress + R_GSTS_REG); } while ((Reg32 & BitMask) == 0); + DEBUG ((DEBUG_INFO, "GSTS_REG : 0x%08x \n", Reg32)); } /** @@ -600,22 +657,19 @@ EnableDmar ( // UpdateRootTableAddressRegister (Index, TRUE); + DEBUG((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); + SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); + DEBUG((DEBUG_INFO, "Enable Abort DMA Mode...\n")); SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_TE); } else { UpdateRootTableAddressRegister (Index, FALSE); + DEBUG((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); + SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); } - DEBUG((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); - SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); - - // - // Init DMAr Fault Event and Data registers - // - MmioRead32 (mVtdUnitInformation[Index].VtdUnitBaseAddress + R_FEDATA_REG); - // // Write Buffer Flush before invalidation // @@ -637,6 +691,9 @@ EnableDmar ( } UpdateRootTableAddressRegister (Index, FALSE); + + DEBUG((DEBUG_INFO, "EnableDmar: waiting for RTPS bit to be set... \n")); + SetGlobalCommandRegisterBits (VtdUnitBaseAddress, B_GMCD_REG_SRTP); } // diff --git a/Silicon/Intel/IntelSiliconPkg/Include/IndustryStandard/Vtd.h b/Silicon/Intel/IntelSiliconPkg/Include/IndustryStandard/Vtd.h index 32fbdd02e..3e1b8c3c5 100644 --- a/Silicon/Intel/IntelSiliconPkg/Include/IndustryStandard/Vtd.h +++ b/Silicon/Intel/IntelSiliconPkg/Include/IndustryStandard/Vtd.h @@ -259,6 +259,8 @@ typedef union { #define R_IQA_REG 0x90 +#define R_IQERCD_REG 0xB0 + #define VTD_PAGE_SHIFT (12) #define VTD_PAGE_SIZE (1UL << VTD_PAGE_SHIFT) #define VTD_PAGE_MASK (((UINT64)-1) << VTD_PAGE_SHIFT) @@ -289,13 +291,20 @@ typedef union { #define QI_IWD_STATUS_WRITE (((UINT64)1) << 5) // -// This is the queued invalidate descriptor. +// queued invalidation 128-bit descriptor // typedef struct { - UINT64 Low; - UINT64 High; + UINT64 Low; + UINT64 High; } QI_DESC; +// +// queued invalidation 256-bit descriptor +// +typedef struct { + UINT64 Uint64[4]; +} QI_256_DESC; + typedef union { struct { UINT8 Minor:4; @@ -411,6 +420,56 @@ typedef union { UINT64 Uint64[2]; } VTD_FRCD_REG; +typedef union { + struct { + UINT32 IQEI:4; // Invalidation Queue Error Info + UINT32 Rsvd_4:28; + UINT32 ITESID:16; // Invalidation Time-out Error Source Identifier + UINT32 ICESID:16; // Invalidation Completion Error Source Identifier + } Bits; + UINT64 Uint64; +} VTD_IQERCD_REG; + +typedef union { + struct { + UINT64 Rsvd_0:4; + UINT64 QH:15; // Queue Head + UINT64 Rsvd_19:45; + } Bits128Desc; + struct { + UINT64 Rsvd_0:4; + UINT64 Rsvd_4:1; + UINT64 QH:14; // Queue Head + UINT64 Rsvd_19:45; + } Bits256Desc; + UINT64 Uint64; +} VTD_IQH_REG; + +typedef union { + struct { + UINT64 Rsvd_0:4; + UINT64 QT:15; // Queue Tail + UINT64 Rsvd_19:45; + } Bits128Desc; + struct { + UINT64 Rsvd_0:4; + UINT64 Rsvd_4:1; + UINT64 QT:14; // Queue Tail + UINT64 Rsvd_19:45; + } Bits256Desc; + UINT64 Uint64; +} VTD_IQT_REG; + +typedef union { + struct { + UINT64 QS:3; // Queue Size + UINT64 Rsvd_3:8; + UINT64 DW:1; // Descriptor Width + UINT64 IQA:52; // Invalidation Queue Base Address + } Bits; + UINT64 Uint64; +} VTD_IQA_REG; + typedef union { struct { UINT8 Function:3; -- 2.26.2.windows.1 -=-=-=-=-=-=-=-=-=-=-=- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#96813): https://edk2.groups.io/g/devel/message/96813 Mute This Topic: https://groups.io/mt/95376272/21656 Group Owner: devel+ow...@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-