Re: [edk2-devel] [PATCH v10 45/46] UefiCpuPkg/MpInitLib: Prepare SEV-ES guest APs for OS use
Reviewed-by: Eric Dong > -Original Message- > From: Tom Lendacky > Sent: Tuesday, July 14, 2020 10:38 PM > To: devel@edk2.groups.io > Cc: Brijesh Singh ; Ard Biesheuvel > ; Dong, Eric ; Justen, > Jordan L ; Laszlo Ersek ; > Gao, Liming ; Kinney, Michael D > ; Ni, Ray > Subject: [PATCH v10 45/46] UefiCpuPkg/MpInitLib: Prepare SEV-ES guest APs > for OS use > > From: Tom Lendacky > > BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2198 > > Before UEFI transfers control to the OS, it must park the AP. This is done > using the AsmRelocateApLoop function to transition into 32-bit non-paging > mode. For an SEV-ES guest, a few additional things must be > done: > - AsmRelocateApLoop must be updated to support SEV-ES. This means > performing a VMGEXIT AP Reset Hold instead of an MWAIT or HLT loop. > - Since the AP must transition to real mode, a small routine is copied > to the WakeupBuffer area. Since the WakeupBuffer will be used by > the AP during OS booting, it must be placed in reserved memory. > Additionally, the AP stack must be located where it can be accessed > in real mode. > - Once the AP is in real mode it will transfer control to the > destination specified by the OS in the SEV-ES AP Jump Table. The > SEV-ES AP Jump Table address is saved by the hypervisor for the OS > using the GHCB VMGEXIT AP Jump Table exit code. > > Cc: Eric Dong > Cc: Ray Ni > Cc: Laszlo Ersek > Signed-off-by: Tom Lendacky > --- > UefiCpuPkg/Library/MpInitLib/MpLib.h | 8 +- > UefiCpuPkg/Library/MpInitLib/DxeMpLib.c | 54 +++- > UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm | 131 > -- > 3 files changed, 175 insertions(+), 18 deletions(-) > > diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.h > b/UefiCpuPkg/Library/MpInitLib/MpLib.h > index b1a9d99cb3eb..267aa5201c50 100644 > --- a/UefiCpuPkg/Library/MpInitLib/MpLib.h > +++ b/UefiCpuPkg/Library/MpInitLib/MpLib.h > @@ -293,7 +293,8 @@ struct _CPU_MP_DATA { >UINT64 GhcbBase; > }; > > -#define AP_RESET_STACK_SIZE 64 > +#define AP_SAFE_STACK_SIZE 128 > +#define AP_RESET_STACK_SIZE AP_SAFE_STACK_SIZE > > #pragma pack(1) > > @@ -349,8 +350,11 @@ VOID >IN BOOLEAN MwaitSupport, >IN UINTN ApTargetCState, >IN UINTN PmCodeSegment, > + IN UINTN Pm16CodeSegment, >IN UINTN TopOfApStack, > - IN UINTN NumberToFinish > + IN UINTN NumberToFinish, > + IN UINTN SevEsAPJumpTable, > + IN UINTN WakeupBuffer >); > > /** > diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c > b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c > index 9115ff9e3e30..7165bcf3124a 100644 > --- a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c > +++ b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c > @@ -12,6 +12,7 @@ > #include > #include > #include > +#include > #include > #include > > @@ -85,6 +86,13 @@ GetWakeupBuffer ( > { >EFI_STATUS Status; >EFI_PHYSICAL_ADDRESSStartAddress; > + EFI_MEMORY_TYPE MemoryType; > + > + if (PcdGetBool (PcdSevEsIsEnabled)) { > +MemoryType = EfiReservedMemoryType; } else { > +MemoryType = EfiBootServicesData; > + } > >// >// Try to allocate buffer below 1M for waking vector. > @@ -97,7 +105,7 @@ GetWakeupBuffer ( >StartAddress = 0x88000; >Status = gBS->AllocatePages ( >AllocateMaxAddress, > - EfiBootServicesData, > + MemoryType, >EFI_SIZE_TO_PAGES (WakeupBufferSize), >&StartAddress >); > @@ -159,8 +167,10 @@ GetSevEsAPMemory ( >VOID >) > { > - EFI_STATUSStatus; > - EFI_PHYSICAL_ADDRESS StartAddress; > + EFI_STATUSStatus; > + EFI_PHYSICAL_ADDRESS StartAddress; > + MSR_SEV_ES_GHCB_REGISTER Msr; > + GHCB *Ghcb; > >// >// Allocate 1 page for AP jump table page @@ -176,6 +186,16 @@ > GetSevEsAPMemory ( > >DEBUG ((DEBUG_INFO, "Dxe: SevEsAPMemory = %lx\n", (UINTN) > StartAddress)); > > + // > + // Save the SevEsAPMemory as the AP jump table. > + // > + Msr.GhcbPhysicalAddress = AsmReadMsr64 (MSR_SEV_ES_GHCB); Ghcb > = > + Msr.Ghcb; > + > + VmgInit (Ghcb); > + VmgExit (Ghcb, SVM_EXIT_AP_JUMP_TABLE, 0, (UINT64) (UINTN) > + StartAddress); VmgDone (Ghcb); > + >return (UINTN) StartAddress; > } > > @@ -330,17 +350,26 @@ RelocateApLoop ( >BOOLEANMwaitSupport; >ASM_RELOCATE_AP_LOOP AsmRelocateApLoopFunc; >UINTN ProcessorNumber; > + UINTN StackStart; > >MpInitLibWhoAmI (&ProcessorNumber); >CpuMpData= GetCpuMpData (); >MwaitSupport = IsMwaitSupport (); > + if (CpuMpData->SevEsIsEnabled) { > +StackStart = CpuMpData->SevEsAPResetStackStart; > + } else
[edk2-devel] [PATCH v10 45/46] UefiCpuPkg/MpInitLib: Prepare SEV-ES guest APs for OS use
From: Tom Lendacky BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2198 Before UEFI transfers control to the OS, it must park the AP. This is done using the AsmRelocateApLoop function to transition into 32-bit non-paging mode. For an SEV-ES guest, a few additional things must be done: - AsmRelocateApLoop must be updated to support SEV-ES. This means performing a VMGEXIT AP Reset Hold instead of an MWAIT or HLT loop. - Since the AP must transition to real mode, a small routine is copied to the WakeupBuffer area. Since the WakeupBuffer will be used by the AP during OS booting, it must be placed in reserved memory. Additionally, the AP stack must be located where it can be accessed in real mode. - Once the AP is in real mode it will transfer control to the destination specified by the OS in the SEV-ES AP Jump Table. The SEV-ES AP Jump Table address is saved by the hypervisor for the OS using the GHCB VMGEXIT AP Jump Table exit code. Cc: Eric Dong Cc: Ray Ni Cc: Laszlo Ersek Signed-off-by: Tom Lendacky --- UefiCpuPkg/Library/MpInitLib/MpLib.h | 8 +- UefiCpuPkg/Library/MpInitLib/DxeMpLib.c | 54 +++- UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm | 131 -- 3 files changed, 175 insertions(+), 18 deletions(-) diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.h b/UefiCpuPkg/Library/MpInitLib/MpLib.h index b1a9d99cb3eb..267aa5201c50 100644 --- a/UefiCpuPkg/Library/MpInitLib/MpLib.h +++ b/UefiCpuPkg/Library/MpInitLib/MpLib.h @@ -293,7 +293,8 @@ struct _CPU_MP_DATA { UINT64 GhcbBase; }; -#define AP_RESET_STACK_SIZE 64 +#define AP_SAFE_STACK_SIZE 128 +#define AP_RESET_STACK_SIZE AP_SAFE_STACK_SIZE #pragma pack(1) @@ -349,8 +350,11 @@ VOID IN BOOLEAN MwaitSupport, IN UINTN ApTargetCState, IN UINTN PmCodeSegment, + IN UINTN Pm16CodeSegment, IN UINTN TopOfApStack, - IN UINTN NumberToFinish + IN UINTN NumberToFinish, + IN UINTN SevEsAPJumpTable, + IN UINTN WakeupBuffer ); /** diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c index 9115ff9e3e30..7165bcf3124a 100644 --- a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c +++ b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -85,6 +86,13 @@ GetWakeupBuffer ( { EFI_STATUS Status; EFI_PHYSICAL_ADDRESSStartAddress; + EFI_MEMORY_TYPE MemoryType; + + if (PcdGetBool (PcdSevEsIsEnabled)) { +MemoryType = EfiReservedMemoryType; + } else { +MemoryType = EfiBootServicesData; + } // // Try to allocate buffer below 1M for waking vector. @@ -97,7 +105,7 @@ GetWakeupBuffer ( StartAddress = 0x88000; Status = gBS->AllocatePages ( AllocateMaxAddress, - EfiBootServicesData, + MemoryType, EFI_SIZE_TO_PAGES (WakeupBufferSize), &StartAddress ); @@ -159,8 +167,10 @@ GetSevEsAPMemory ( VOID ) { - EFI_STATUSStatus; - EFI_PHYSICAL_ADDRESS StartAddress; + EFI_STATUSStatus; + EFI_PHYSICAL_ADDRESS StartAddress; + MSR_SEV_ES_GHCB_REGISTER Msr; + GHCB *Ghcb; // // Allocate 1 page for AP jump table page @@ -176,6 +186,16 @@ GetSevEsAPMemory ( DEBUG ((DEBUG_INFO, "Dxe: SevEsAPMemory = %lx\n", (UINTN) StartAddress)); + // + // Save the SevEsAPMemory as the AP jump table. + // + Msr.GhcbPhysicalAddress = AsmReadMsr64 (MSR_SEV_ES_GHCB); + Ghcb = Msr.Ghcb; + + VmgInit (Ghcb); + VmgExit (Ghcb, SVM_EXIT_AP_JUMP_TABLE, 0, (UINT64) (UINTN) StartAddress); + VmgDone (Ghcb); + return (UINTN) StartAddress; } @@ -330,17 +350,26 @@ RelocateApLoop ( BOOLEANMwaitSupport; ASM_RELOCATE_AP_LOOP AsmRelocateApLoopFunc; UINTN ProcessorNumber; + UINTN StackStart; MpInitLibWhoAmI (&ProcessorNumber); CpuMpData= GetCpuMpData (); MwaitSupport = IsMwaitSupport (); + if (CpuMpData->SevEsIsEnabled) { +StackStart = CpuMpData->SevEsAPResetStackStart; + } else { +StackStart = mReservedTopOfApStack; + } AsmRelocateApLoopFunc = (ASM_RELOCATE_AP_LOOP) (UINTN) mReservedApLoopFunc; AsmRelocateApLoopFunc ( MwaitSupport, CpuMpData->ApTargetCState, CpuMpData->PmCodeSegment, -mReservedTopOfApStack - ProcessorNumber * AP_SAFE_STACK_SIZE, -(UINTN) &mNumberToFinish +CpuMpData->Pm16CodeSegment, +StackStart - ProcessorNumber * AP_SAFE_STACK_SIZE, +(UINTN) &mNumberToFinish, +CpuMpData->SevEsAPBuffer, +CpuMpData->WakeupBuffer ); // // It should never reach here @@ -374,6 +403,21 @@ MpInitChangeApLoopCallback ( while (mNumberToFini