Tested-by: Yuanhao Xie <[email protected]>
Signed-off-by: Yuanhao Xie <[email protected]>
---
UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf | 6 +++++-
UefiCpuPkg/Library/MpInitLib/DxeMpLib.c | 28
++++++++++++++++++----------
UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c | 23
+++++++++++++++++++++++
UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm | 11 ++++-------
UefiCpuPkg/Library/MpInitLib/MpEqu.inc | 22
+++++++++++-----------
UefiCpuPkg/Library/MpInitLib/MpLib.h | 17 +++++++++++++----
UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c | 82
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm | 173
++++++++++++++++++++++++++++-------------------------------------------------------------------------------------------------------------------------------------------------
UefiCpuPkg/UefiCpuPkg.dsc | 3 ++-
9 files changed, 186 insertions(+), 179 deletions(-)
diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
index cd07de3a3c..4285dd06b4 100644
--- a/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
+++ b/UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
@@ -1,7 +1,7 @@
## @file
# MP Initialize Library instance for DXE driver.
#
-# Copyright (c) 2016 - 2021, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2016 - 2023, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
##
@@ -24,10 +24,12 @@
[Sources.IA32]
Ia32/AmdSev.c
Ia32/MpFuncs.nasm
+ Ia32/CreatePageTable.c
[Sources.X64]
X64/AmdSev.c
X64/MpFuncs.nasm
+ X64/CreatePageTable.c
[Sources.common]
AmdSev.c
@@ -56,6 +58,8 @@
PcdLib
CcExitLib
MicrocodeLib
+[LibraryClasses.X64]
+ CpuPageTableLib
[Protocols]
gEfiTimerArchProtocolGuid ## SOMETIMES_CONSUMES
diff --git a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
index 75743faf5f..76953206ca 100644
--- a/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
+++ b/UefiCpuPkg/Library/MpInitLib/DxeMpLib.c
@@ -28,7 +28,7 @@ volatile BOOLEAN mStopCheckAllApsStatus = TRUE;
UINTN mReservedTopOfApStack;
volatile UINT32 mNumberToFinish = 0;
RELOCATE_AP_LOOP_ENTRY mReservedApLoop;
-
+UINTN mApPageTable;
//
// Begin wakeup buffer allocation below 0x88000
@@ -379,10 +379,10 @@ RelocateApLoop (
IN OUT VOID *Buffer
)
{
- CPU_MP_DATA *CpuMpData;
- BOOLEAN MwaitSupport;
- UINTN ProcessorNumber;
- UINTN StackStart;
+ CPU_MP_DATA *CpuMpData;
+ BOOLEAN MwaitSupport;
+ UINTN ProcessorNumber;
+ UINTN StackStart;
MpInitLibWhoAmI (&ProcessorNumber);
CpuMpData = GetCpuMpData ();
@@ -404,12 +404,9 @@ RelocateApLoop (
mReservedApLoop.GenericEntry (
MwaitSupport,
CpuMpData->ApTargetCState,
- CpuMpData->PmCodeSegment,
StackStart - ProcessorNumber * AP_SAFE_STACK_SIZE,
(UINTN)&mNumberToFinish,
- CpuMpData->Pm16CodeSegment,
- CpuMpData->SevEsAPBuffer,
- CpuMpData->WakeupBuffer
+ mApPageTable
);
}
@@ -540,9 +537,11 @@ InitMpGlobalData (
AddressMap = &CpuMpData->AddressMap;
if (CpuMpData->UseSevEsAPMethod) {
+ Address = BASE_4GB - 1;
ApLoopFunc = AddressMap->RelocateApLoopFuncAddressAmdSev;
ApLoopFuncSize = AddressMap->RelocateApLoopFuncSizeAmdSev;
} else {
+ Address = MAX_ADDRESS;
ApLoopFunc = AddressMap->RelocateApLoopFuncAddress;
ApLoopFuncSize = AddressMap->RelocateApLoopFuncSize;
}
@@ -564,7 +563,6 @@ InitMpGlobalData (
// +------------+ (low address )
//
- Address = BASE_4GB - 1;
StackPages = EFI_SIZE_TO_PAGES (CpuMpData->CpuCount * AP_SAFE_STACK_SIZE);
FuncPages = EFI_SIZE_TO_PAGES (ALIGN_VALUE (ApLoopFuncSize,
EFI_PAGE_SIZE));
@@ -597,6 +595,16 @@ InitMpGlobalData (
ASSERT ((mReservedTopOfApStack & (UINTN)(CPU_STACK_ALIGNMENT - 1)) == 0);
mReservedApLoop.Data = (VOID *)(UINTN)Address;
CopyMem (mReservedApLoop.Data, ApLoopFunc, ApLoopFuncSize);
+ if (!CpuMpData->UseSevEsAPMethod) {
+ //
+ // non-Sev Processor
+ //
+ mApPageTable = CreatePageTable (
+ (UINTN)Address,
+ EFI_PAGES_TO_SIZE (StackPages+FuncPages)
+ );
+ }
+
Status = gBS->CreateEvent (
EVT_TIMER | EVT_NOTIFY_SIGNAL,
TPL_NOTIFY,
diff --git a/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c
b/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c
new file mode 100644
index 0000000000..bec9b247c0
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/Ia32/CreatePageTable.c
@@ -0,0 +1,23 @@
+/** @file
+ Function to create page talbe.
+ Only create page table for x64, and leave the CreatePageTable empty for Ia32.
+ Copyright (c) 2023, Intel Corporation. All rights reserved.<BR>
+ SPDX-License-Identifier: BSD-2-Clause-Patent
+**/
+
+#include <Base.h>
+
+/**
+ Only create page table for x64, and leave the CreatePageTable empty for Ia32.
+ @param[in] LinearAddress The start of the linear address range.
+ @param[in] Length The length of the linear address range.
+ @return The page table to be created.
+**/
+UINTN
+CreatePageTable (
+ IN UINTN Address,
+ IN UINTN Length
+ )
+{
+ return 0;
+}
diff --git a/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm
b/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm
index bfcdbd31c1..c65a825a23 100644
--- a/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm
+++ b/UefiCpuPkg/Library/MpInitLib/Ia32/MpFuncs.nasm
@@ -1,5 +1,5 @@
;------------------------------------------------------------------------------
;
-; Copyright (c) 2015 - 2022, Intel Corporation. All rights reserved.<BR>
+; Copyright (c) 2015 - 2023, Intel Corporation. All rights reserved.<BR>
; SPDX-License-Identifier: BSD-2-Clause-Patent
;
; Module Name:
@@ -219,20 +219,17 @@ SwitchToRealProcEnd:
RendezvousFunnelProcEnd:
;-------------------------------------------------------------------------------------
-; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment,
TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
-;
-; The last three parameters (Pm16CodeSegment, SevEsAPJumpTable and
WakeupBuffer) are
-; specific to SEV-ES support and are not applicable on IA32.
+; AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack,
CountTofinish, Cr3);
;-------------------------------------------------------------------------------------
AsmRelocateApLoopStart:
mov eax, esp
- mov esp, [eax + 16] ; TopOfApStack
+ mov esp, [eax + 12] ; TopOfApStack
push dword [eax] ; push return address for stack trace
push ebp
mov ebp, esp
mov ebx, [eax + 8] ; ApTargetCState
mov ecx, [eax + 4] ; MwaitSupport
- mov eax, [eax + 20] ; CountTofinish
+ mov eax, [eax + 16] ; CountTofinish
lock dec dword [eax] ; (*CountTofinish)--
cmp cl, 1 ; Check mwait-monitor support
jnz HltLoop
diff --git a/UefiCpuPkg/Library/MpInitLib/MpEqu.inc
b/UefiCpuPkg/Library/MpInitLib/MpEqu.inc
index 1472ef2024..6730f2f411 100644
--- a/UefiCpuPkg/Library/MpInitLib/MpEqu.inc
+++ b/UefiCpuPkg/Library/MpInitLib/MpEqu.inc
@@ -21,17 +21,17 @@ CPU_SWITCH_STATE_LOADED equ 2
; Equivalent NASM structure of MP_ASSEMBLY_ADDRESS_MAP
;
struc MP_ASSEMBLY_ADDRESS_MAP
- .RendezvousFunnelAddress CTYPE_UINTN 1
- .ModeEntryOffset CTYPE_UINTN 1
- .RendezvousFunnelSize CTYPE_UINTN 1
- .RelocateApLoopFuncAddress CTYPE_UINTN 1
- .RelocateApLoopFuncSize CTYPE_UINTN 1
- .RelocateApLoopFuncAddressAmdSev CTYPE_UINTN 1
- .RelocateApLoopFuncSizeAmdSev CTYPE_UINTN 1
- .ModeTransitionOffset CTYPE_UINTN 1
- .SwitchToRealNoNxOffset CTYPE_UINTN 1
- .SwitchToRealPM16ModeOffset CTYPE_UINTN 1
- .SwitchToRealPM16ModeSize CTYPE_UINTN 1
+ .RendezvousFunnelAddress CTYPE_UINTN 1
+ .ModeEntryOffset CTYPE_UINTN 1
+ .RendezvousFunnelSize CTYPE_UINTN 1
+ .RelocateApLoopFuncAddress CTYPE_UINTN 1
+ .RelocateApLoopFuncSize CTYPE_UINTN 1
+ .RelocateApLoopFuncAddressAmdSev CTYPE_UINTN 1
+ .RelocateApLoopFuncSizeAmdSev CTYPE_UINTN 1
+ .ModeTransitionOffset CTYPE_UINTN 1
+ .SwitchToRealNoNxOffset CTYPE_UINTN 1
+ .SwitchToRealPM16ModeOffset CTYPE_UINTN 1
+ .SwitchToRealPM16ModeSize CTYPE_UINTN 1
endstruc
;
diff --git a/UefiCpuPkg/Library/MpInitLib/MpLib.h
b/UefiCpuPkg/Library/MpInitLib/MpLib.h
index 772a828045..cc77843bea 100644
--- a/UefiCpuPkg/Library/MpInitLib/MpLib.h
+++ b/UefiCpuPkg/Library/MpInitLib/MpLib.h
@@ -368,12 +368,9 @@ typedef
(EFIAPI *ASM_RELOCATE_AP_LOOP)(
IN BOOLEAN MwaitSupport,
IN UINTN ApTargetCState,
- IN UINTN PmCodeSegment,
IN UINTN TopOfApStack,
IN UINTN NumberToFinish,
- IN UINTN Pm16CodeSegment,
- IN UINTN SevEsAPJumpTable,
- IN UINTN WakeupBuffer
+ IN UINTN Cr3
);
/**
@@ -498,6 +495,18 @@ GetSevEsAPMemory (
VOID
);
+/**
+ Create 1:1 mapping page table in reserved memory to map the specified
address range.
+ @param[in] LinearAddress The start of the linear address range.
+ @param[in] Length The length of the linear address range.
+ @return The page table to be created.
+**/
+UINTN
+CreatePageTable (
+ IN UINTN Address,
+ IN UINTN Length
+ );
+
/**
This function will be called by BSP to wakeup AP.
diff --git a/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c b/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c
new file mode 100644
index 0000000000..7cf91ed9c4
--- /dev/null
+++ b/UefiCpuPkg/Library/MpInitLib/X64/CreatePageTable.c
@@ -0,0 +1,82 @@
+/** @file
+ Function to create page talbe.
+ Only create page table for x64, and leave the CreatePageTable empty for Ia32.
+ Copyright (c) 2023, Intel Corporation. All rights reserved.<BR>
+ SPDX-License-Identifier: BSD-2-Clause-Patent
+**/
+#include <Library/CpuPageTableLib.h>
+#include <Library/MemoryAllocationLib.h>
+#include <Base.h>
+#include <Library/BaseMemoryLib.h>
+#include <Library/DebugLib.h>
+#include <Library/BaseLib.h>
+
+/**
+ Create 1:1 mapping page table in reserved memory to map the specified
address range.
+ @param[in] LinearAddress The start of the linear address range.
+ @param[in] Length The length of the linear address range.
+ @return The page table to be created.
+**/
+UINTN
+CreatePageTable (
+ IN UINTN Address,
+ IN UINTN Length
+ )
+{
+ EFI_STATUS Status;
+ VOID *PageTableBuffer;
+ UINTN PageTableBufferSize;
+ UINTN PageTable;
+ PAGING_MODE PagingMode;
+ IA32_CR4 Cr4;
+
+ IA32_MAP_ATTRIBUTE MapAttribute;
+ IA32_MAP_ATTRIBUTE MapMask;
+
+ MapAttribute.Uint64 = Address;
+ MapAttribute.Bits.Present = 1;
+ MapAttribute.Bits.ReadWrite = 1;
+
+ MapMask.Bits.PageTableBaseAddress = 1;
+ MapMask.Bits.Present = 1;
+ MapMask.Bits.ReadWrite = 1;
+
+ PageTable = 0;
+ PageTableBufferSize = 0;
+
+ Cr4.UintN = AsmReadCr4 ();
+
+ if (Cr4.Bits.LA57 == 1) {
+ PagingMode = Paging5Level;
+ } else {
+ PagingMode = Paging4Level;
+ }
+
+ Status = PageTableMap (
+ &PageTable,
+ PagingMode,
+ NULL,
+ &PageTableBufferSize,
+ Address,
+ Length,
+ &MapAttribute,
+ &MapMask
+ );
+ ASSERT (Status == EFI_BUFFER_TOO_SMALL);
+ DEBUG ((DEBUG_INFO, "AP Page Table Buffer Size = %x\n",
PageTableBufferSize));
+
+ PageTableBuffer = AllocateReservedPages (EFI_SIZE_TO_PAGES
(PageTableBufferSize));
+ ASSERT (PageTableBuffer != NULL);
+ Status = PageTableMap (
+ &PageTable,
+ PagingMode,
+ PageTableBuffer,
+ &PageTableBufferSize,
+ Address,
+ Length,
+ &MapAttribute,
+ &MapMask
+ );
+ ASSERT_EFI_ERROR (Status);
+ return PageTable;
+}
diff --git a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm
b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm
index d36f8ba06d..2bce04d99c 100644
--- a/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm
+++ b/UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm
@@ -279,172 +279,55 @@ CProcedureInvoke:
RendezvousFunnelProcEnd:
;-------------------------------------------------------------------------------------
-; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment,
TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
+; AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack,
CountTofinish, Cr3);
+; This function is called during the finalizaiton of Mp initialization before
booting
+; to OS, and aim to put Aps either in Mwait or HLT.
;-------------------------------------------------------------------------------------
-AsmRelocateApLoopStart:
-BITS 64
- cmp qword [rsp + 56], 0 ; SevEsAPJumpTable
- je NoSevEs
-
- ;
- ; Perform some SEV-ES related setup before leaving 64-bit mode
- ;
- push rcx
- push rdx
-
- ;
- ; Get the RDX reset value using CPUID
- ;
- mov rax, 1
- cpuid
- mov rsi, rax ; Save off the reset value for RDX
-
- ;
- ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
- ; - Must be done while in 64-bit long mode so that writes to
- ; the GHCB memory will be unencrypted.
- ; - No NAE events can be generated once this is set otherwise
- ; the AP_RESET_HOLD SW_EXITCODE will be overwritten.
- ;
- mov rcx, 0xc0010130
- rdmsr ; Retrieve current GHCB address
- shl rdx, 32
- or rdx, rax
-
- mov rdi, rdx
- xor rax, rax
- mov rcx, 0x800
- shr rcx, 3
- rep stosq ; Clear the GHCB
-
- mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD
- mov [rdx + 0x390], rax
- mov rax, 114 ; Set SwExitCode valid bit
- bts [rdx + 0x3f0], rax
- inc rax ; Set SwExitInfo1 valid bit
- bts [rdx + 0x3f0], rax
- inc rax ; Set SwExitInfo2 valid bit
- bts [rdx + 0x3f0], rax
+; +----------------+
+; | Cr3 | rsp+40
+; +----------------+
+; | CountTofinish | r9
+; +----------------+
+; | TopOfApStack | r8
+; +----------------+
+; | ApTargetCState | rdx
+; +----------------+
+; | MwaitSupport | rcx
+; +----------------+
+; | the return |
+; +----------------+ low address
- pop rdx
- pop rcx
-
-NoSevEs:
- cli ; Disable interrupt before switching to
32-bit mode
- mov rax, [rsp + 40] ; CountTofinish
+AsmRelocateApLoopStart:
+ mov rax, r9 ; CountTofinish
lock dec dword [rax] ; (*CountTofinish)--
- mov r10, [rsp + 48] ; Pm16CodeSegment
- mov rax, [rsp + 56] ; SevEsAPJumpTable
- mov rbx, [rsp + 64] ; WakeupBuffer
- mov rsp, r9 ; TopOfApStack
-
- push rax ; Save SevEsAPJumpTable
- push rbx ; Save WakeupBuffer
- push r10 ; Save Pm16CodeSegment
- push rcx ; Save MwaitSupport
- push rdx ; Save ApTargetCState
-
- lea rax, [PmEntry] ; rax <- The start address of transition code
-
- push r8
- push rax
-
- ;
- ; Clear R8 - R15, for reset, before going into 32-bit mode
- ;
- xor r8, r8
- xor r9, r9
- xor r10, r10
- xor r11, r11
- xor r12, r12
- xor r13, r13
- xor r14, r14
- xor r15, r15
-
- ;
- ; Far return into 32-bit mode
- ;
- retfq
-
-BITS 32
-PmEntry:
- mov eax, cr0
- btr eax, 31 ; Clear CR0.PG
- mov cr0, eax ; Disable paging and caches
-
- mov ecx, 0xc0000080
- rdmsr
- and ah, ~ 1 ; Clear LME
- wrmsr
- mov eax, cr4
- and al, ~ (1 << 5) ; Clear PAE
- mov cr4, eax
-
- pop edx
- add esp, 4
- pop ecx,
- add esp, 4
+ mov rax, [rsp + 40] ; Cr3
+ ; Do not push on old stack, since old stack is not mapped
+ ; in the page table pointed by cr3
+ mov cr3, rax
+ mov rsp, r8 ; TopOfApStack
MwaitCheck:
cmp cl, 1 ; Check mwait-monitor support
jnz HltLoop
- mov ebx, edx ; Save C-State to ebx
+ mov rbx, rdx ; Save C-State to ebx
+
MwaitLoop:
cli
- mov eax, esp ; Set Monitor Address
+ mov rax, rsp ; Set Monitor Address
xor ecx, ecx ; ecx = 0
xor edx, edx ; edx = 0
monitor
- mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]
+ mov rax, rbx ; Mwait Cx, Target C-State per eax[7:4]
shl eax, 4
mwait
jmp MwaitLoop
HltLoop:
- pop edx ; PM16CodeSegment
- add esp, 4
- pop ebx ; WakeupBuffer
- add esp, 4
- pop eax ; SevEsAPJumpTable
- add esp, 4
- cmp eax, 0 ; Check for SEV-ES
- je DoHlt
-
- cli
- ;
- ; SEV-ES is enabled, use VMGEXIT (GHCB information already
- ; set by caller)
- ;
-BITS 64
- rep vmmcall
-BITS 32
-
- ;
- ; Back from VMGEXIT AP_HLT_LOOP
- ; Push the FLAGS/CS/IP values to use
- ;
- push word 0x0002 ; EFLAGS
- xor ecx, ecx
- mov cx, [eax + 2] ; CS
- push cx
- mov cx, [eax] ; IP
- push cx
- push word 0x0000 ; For alignment, will be discarded
-
- push edx
- push ebx
-
- mov edx, esi ; Restore RDX reset value
-
- retf
-
-DoHlt:
cli
hlt
- jmp DoHlt
+ jmp HltLoop
-BITS 64
AsmRelocateApLoopEnd:
;-------------------------------------------------------------------------------------
diff --git a/UefiCpuPkg/UefiCpuPkg.dsc b/UefiCpuPkg/UefiCpuPkg.dsc
index f9a46089d2..062a43d7a7 100644
--- a/UefiCpuPkg/UefiCpuPkg.dsc
+++ b/UefiCpuPkg/UefiCpuPkg.dsc
@@ -1,7 +1,7 @@
## @file
# UefiCpuPkg Package
#
-# Copyright (c) 2007 - 2022, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2007 - 2023, Intel Corporation. All rights reserved.<BR>
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
@@ -94,6 +94,7 @@
MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf
HobLib|MdePkg/Library/DxeHobLib/DxeHobLib.inf
CpuExceptionHandlerLib|UefiCpuPkg/Library/CpuExceptionHandlerLib/DxeCpuExceptionHandlerLib.inf
+ CpuPageTableLib|UefiCpuPkg/Library/CpuPageTableLib/CpuPageTableLib.inf
MpInitLib|UefiCpuPkg/Library/MpInitLib/DxeMpInitLib.inf
RegisterCpuFeaturesLib|UefiCpuPkg/Library/RegisterCpuFeaturesLib/DxeRegisterCpuFeaturesLib.inf
CpuCacheInfoLib|UefiCpuPkg/Library/CpuCacheInfoLib/DxeCpuCacheInfoLib.inf
