The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=6e93f5e4d6932c423b89dff8fc08d86f8bdeb7b9
commit 6e93f5e4d6932c423b89dff8fc08d86f8bdeb7b9 Author: Konstantin Belousov <[email protected]> AuthorDate: 2026-02-13 11:24:40 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2026-07-20 21:51:02 +0000 amd64: FRED support FRED support as defined starting from the SDM rev. 90, requires a new 'events' entry point to receive user and kernel mode exceptions and interrupts notifications from the hardware. A minimal asm trampoline is enough, rest can be implemented in C due to the clean FRED organization of the event reporting. The syscall entry is handled by a microptimized assembly path, directly calling into the amd64_syscall() handler, instead of the generic events entry point. Tested by: emaste Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D55829 --- sys/amd64/amd64/cpu_switch.S | 73 ++++- sys/amd64/amd64/db_trace.c | 5 +- sys/amd64/amd64/exception.S | 282 ++++++++++++++++++ sys/amd64/amd64/genassym.c | 15 + sys/amd64/amd64/initcpu.c | 43 +++ sys/amd64/amd64/machdep.c | 104 +++++-- sys/amd64/amd64/mp_machdep.c | 82 +++-- sys/amd64/amd64/trap.c | 558 +++++++++++++++++++++++++++++++++++ sys/amd64/amd64/vm_machdep.c | 10 +- sys/amd64/include/md_var.h | 2 + sys/amd64/include/pcb.h | 8 +- sys/amd64/vmm/intel/vmx.c | 2 +- sys/amd64/vmm/vmm.c | 2 +- sys/amd64/vmm/vmm_lapic.c | 5 + sys/amd64/vmm/vmm_lapic.h | 2 + sys/dev/hyperv/vmbus/x86/vmbus_x86.c | 3 +- sys/i386/i386/machdep.c | 2 + sys/kern/kern_thread.c | 2 +- sys/x86/include/apicvar.h | 3 +- sys/x86/include/frame.h | 1 - sys/x86/include/x86_var.h | 3 + sys/x86/x86/local_apic.c | 38 +-- 22 files changed, 1158 insertions(+), 87 deletions(-) diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index 21e55e64d74a..774098d7def5 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -187,14 +187,22 @@ do_kthread: cmpq %rax,%rdx jne do_tss done_tss: + movq %r8,PCPU(CURPCB) + cmpb $0,fred + jne 1f movq TD_MD_STACK_BASE(%r12),%r9 movq %r9,PCPU(RSP0) - movq %r8,PCPU(CURPCB) movq PCPU(PTI_RSP0),%rax cmpq $~0,PCPU(UCR3) cmove %r9,%rax movq %rax,TSS_RSP0(%rdx) - movq %r12,PCPU(CURTHREAD) /* into next thread */ + jmp 2f +1: movq TD_MD_STACK_BASE(%r12),%rdx + movl %edx,%eax + shrq $32,%rdx + movl $MSR_FRED_RSP0,%ecx + wrmsr +2: movq %r12,PCPU(CURTHREAD) /* into next thread */ /* Test if debug registers should be restored. */ testl $PCB_DBREGS,PCB_FLAGS(%r8) @@ -365,8 +373,36 @@ ENTRY(savectx) movl %eax,PCB_SFMASK(%rdi) movl %edx,PCB_SFMASK+4(%rdi) - sgdt PCB_GDT(%rdi) - sidt PCB_IDT(%rdi) + cmpb $0, fred + je 1f + movl $MSR_FRED_RSP0,%ecx + rdmsr + movl %eax,PCB_FRED_RSP0(%rdi) + movl %edx,PCB_FRED_RSP0+4(%rdi) + movl $MSR_FRED_RSP1,%ecx + rdmsr + movl %eax,PCB_FRED_RSP1(%rdi) + movl %edx,PCB_FRED_RSP1+4(%rdi) + movl $MSR_FRED_RSP2,%ecx + rdmsr + movl %eax,PCB_FRED_RSP2(%rdi) + movl %edx,PCB_FRED_RSP2+4(%rdi) + movl $MSR_FRED_RSP3,%ecx + rdmsr + movl %eax,PCB_FRED_RSP3(%rdi) + movl %edx,PCB_FRED_RSP3+4(%rdi) + movl $MSR_FRED_STKLVLS,%ecx + rdmsr + movl %eax,PCB_FRED_STKLVLS(%rdi) + movl %edx,PCB_FRED_STKLVLS+4(%rdi) + movl $MSR_FRED_CONFIG,%ecx + rdmsr + movl %eax,PCB_FRED_CONFIG(%rdi) + movl %edx,PCB_FRED_CONFIG+4(%rdi) + jmp 2f + +1: sidt PCB_IDT(%rdi) +2: sgdt PCB_GDT(%rdi) sldt PCB_LDT(%rdi) str PCB_TR(%rdi) @@ -442,8 +478,35 @@ ENTRY(resumectx) movq %rax,%cr3 /* Restore descriptor tables. */ + cmpb $0,fred + jne 1f lidt PCB_IDT(%rdi) - lldt PCB_LDT(%rdi) + jmp 2f +1: movl $MSR_FRED_RSP0,%ecx + movl PCB_FRED_RSP0(%rdi),%eax + movl 4 + PCB_FRED_RSP0(%rdi),%edx + wrmsr + movl $MSR_FRED_RSP1,%ecx + movl PCB_FRED_RSP1(%rdi),%eax + movl 4 + PCB_FRED_RSP1(%rdi),%edx + wrmsr + movl $MSR_FRED_RSP2,%ecx + movl PCB_FRED_RSP2(%rdi),%eax + movl 4 + PCB_FRED_RSP2(%rdi),%edx + wrmsr + movl $MSR_FRED_RSP3,%ecx + movl PCB_FRED_RSP3(%rdi),%eax + movl 4 + PCB_FRED_RSP3(%rdi),%edx + wrmsr + movl $MSR_FRED_STKLVLS,%ecx + movl PCB_FRED_STKLVLS(%rdi),%eax + movl 4 + PCB_FRED_STKLVLS(%rdi),%edx + wrmsr + movl $MSR_FRED_CONFIG,%ecx + movl PCB_FRED_CONFIG(%rdi),%eax + movl 4 + PCB_FRED_CONFIG(%rdi),%edx + wrmsr +2: lldt PCB_LDT(%rdi) #define SDT_SYSTSS 9 #define SDT_SYSBSY 11 diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c index 5f751906b7e6..e14cd55978de 100644 --- a/sys/amd64/amd64/db_trace.c +++ b/sys/amd64/amd64/db_trace.c @@ -167,7 +167,10 @@ db_nextframe(db_addr_t *fp, db_addr_t *ip, struct thread *td) strcmp(name, "fork_trampoline") == 0 || strcmp(name, "mchk_calltrap") == 0 || strcmp(name, "nmi_calltrap") == 0 || - strcmp(name, "Xdblfault") == 0) + strcmp(name, "Xdblfault") == 0 || + strcmp(name, "fred_events_handlers") == 0 || + strcmp(name, "user_fred_syscall") == 0 || + strcmp(name, "user_fred_non_syscall") == 0) frame_type = TRAP; else if (strncmp(name, "Xatpic_intr", 11) == 0 || strncmp(name, "Xapic_isr", 9) == 0 || diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index 4d58e07caf4b..6dfb8a431fe7 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -1063,6 +1063,8 @@ ENTRY(fork_trampoline) movq %rbx,%rsi /* arg1 */ movq %rsp,%rdx /* trapframe pointer */ call fork_exit + cmpb $0, fred + jne fred_checkast jmp doreti /* Handle any ASTs */ /* @@ -1401,6 +1403,286 @@ gsbase_load_fault: movq $0,PCB_GSBASE(%r8) jmp doreti +/* + * FRED guarantees that on entry, %rflags is set to '2', which in + * particular means that interrupts are disabled, and DF/AC are clear. + * Interrupts are re-enabled by the C handler, if appropriate. + */ + ALIGN_TEXT +user_fred_syscall: + movq %rdx,TF_RDX(%rsp) + movq %rcx,TF_RCX(%rsp) + SAVE_SEGS + movq PCPU(CURPCB),%r11 + andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) + sti + movw $KUDSEL,TF_SS(%rsp) + movw $KUCSEL,TF_CS(%rsp) + movq %r8,TF_R8(%rsp) + movq %r9,TF_R9(%rsp) + movq %rbx,TF_RBX(%rsp) + movq %rbp,TF_RBP(%rsp) + movq %r12,TF_R12(%rsp) + movq %r13,TF_R13(%rsp) + movq %r14,TF_R14(%rsp) + movq %r15,TF_R15(%rsp) + movl $TF_HASSEGS,TF_FLAGS(%rsp) + andl $TF_FRED_EVINFO2_INSTLENMASK,%eax + shrl $TF_FRED_EVINFO2_INSTLENSHIFT,%eax + movq %rax,TF_ERR(%rsp) + movq PCPU(CURTHREAD),%rdi + movl TF_RFLAGS(%rsp),%esi + andl $PSL_T,%esi + call amd64_syscall + movq PCPU(CURPCB),%rax + cli + testl $PCB_FULL_IRET,PCB_FLAGS(%rax) + jnz fred_checkast + movq PCPU(CURTHREAD),%rax + cmpl $0,TD_AST(%rax) + jnz fred_checkast + movq TF_RDI(%rsp),%rdi + movq TF_RSI(%rsp),%rsi + movq TF_RDX(%rsp),%rdx + movq TF_RAX(%rsp),%rax + xorl %r8d,%r8d + xorl %r9d,%r9d + xorl %r10d,%r10d + xorl %r11d,%r11d + addq $TF_ERR,%rsp + .globl fred_syscall_eretu +fred_syscall_eretu: + eretu + + ALIGN_TEXT +user_fred_non_syscall: + movq %rdx,TF_RDX(%rsp) + movq %rcx,TF_RCX(%rsp) + movq %r8,TF_R8(%rsp) + movq %r9,TF_R9(%rsp) + movq %rbx,TF_RBX(%rsp) + movq %rbp,TF_RBP(%rsp) + movq %r10,TF_R10(%rsp) + movq %r11,TF_R11(%rsp) + movq %r12,TF_R12(%rsp) + movq %r13,TF_R13(%rsp) + movq %r14,TF_R14(%rsp) + movq %r15,TF_R15(%rsp) + SAVE_SEGS + movq PCPU(CURTHREAD),%r9 + movq TD_PCB(%r9),%r8 + rdfsbase %rax + movq %rax,PCB_FSBASE(%r8) + rdgsbase %rax + movq %rax,PCB_GSBASE(%r8) + movl $TF_HASSEGS,TF_FLAGS(%rsp) + KMSAN_ENTER + movq %rsp, %rdi + call trap_fred_u + KMSAN_LEAVE +fred_checkast: + cli + movq PCPU(CURTHREAD),%rax + movq TD_PCB(%rax),%r8 + cmpl $0,TD_AST(%rax) + je fred_gouser + sti + movq %rsp,%rdi /* pass a pointer to the trapframe */ + call ast + jmp fred_checkast +fred_gouser: + testl $PCB_FULL_IRET,PCB_FLAGS(%r8) + jz fred_u_regs + andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) + testl $TF_HASSEGS,TF_FLAGS(%rsp) + jne fred_do_segs + movw $KUDSEL,%ax + movw %ax,TF_DS(%rsp) + movw %ax,TF_ES(%rsp) + movw $KUF32SEL,TF_FS(%rsp) + movw $KUG32SEL,TF_GS(%rsp) +fred_do_segs: + /* Restore %fs and fsbase */ + movw TF_FS(%rsp),%ax + .globl fred_ld_fs +fred_ld_fs: + movw %ax,%fs + testl $PCB_32BIT,PCB_FLAGS(%r8) + jnz after_fred_fs + movq PCB_FSBASE(%r8),%rax + .globl fred_ld_fsbase +fred_ld_fsbase: + wrfsbase %rax +after_fred_fs: + /* Restore %gs and kgsbase */ + movw TF_GS(%rsp),%ax + .globl fred_lkgs +fred_lkgs: + lkgs %ax + testl $PCB_32BIT,PCB_FLAGS(%r8) + jnz after_fred_gs + movl PCB_GSBASE(%r8),%eax + movl PCB_GSBASE+4(%r8),%edx + movl $MSR_KGSBASE,%ecx + .globl fred_ld_kgsbase +fred_ld_kgsbase: + wrmsr +after_fred_gs: + .globl fred_ld_es +fred_ld_es: + movw TF_ES(%rsp),%es + .globl fred_ld_ds +fred_ld_ds: + movw TF_DS(%rsp),%ds +fred_u_regs: + RESTORE_REGS + addq $TF_ERR,%rsp + .globl fred_eretu +fred_eretu: + eretu + + .globl fred_ld_fs_fault +fred_ld_fs_fault: + movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp) + movq %rsp,%rdi + call trap_fred_u + movw $KUF32SEL,TF_FS(%rsp) + jmp fred_checkast + + .globl fred_ld_fsbase_fault +fred_ld_fsbase_fault: + movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp) + movq %rsp,%rdi + call trap_fred_u + movq PCPU(CURTHREAD),%r9 + movq TD_PCB(%r9),%r8 + movq $0,PCB_FSBASE(%r8) + jmp fred_checkast + + .globl fred_lkgs_fault +fred_lkgs_fault: + movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp) + movq %rsp,%rdi + call trap_fred_u + movw $KUG32SEL,TF_GS(%rsp) + jmp fred_checkast + + .globl fred_ld_kgsbase_fault +fred_ld_kgsbase_fault: + movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp) + movq %rsp,%rdi + call trap_fred_u + movq PCPU(CURTHREAD),%r9 + movq TD_PCB(%r9),%r8 + movq $0,PCB_GSBASE(%r8) + jmp fred_checkast + + .globl fred_ld_es_fault +fred_ld_es_fault: + movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp) + movq %rsp,%rdi + call trap_fred_u + movw $KUDSEL,TF_ES(%rsp) + jmp fred_checkast + + .globl fred_ld_ds_fault +fred_ld_ds_fault: + movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp) + movq %rsp,%rdi + call trap_fred_u + movw $KUDSEL,TF_DS(%rsp) + jmp fred_checkast + + .globl fred_eretu_fault +fred_eretu_fault: + subq $TF_ERR,%rsp + movq %rdi,TF_RDI(%rsp) + movq %rdx,TF_RDX(%rsp) + movq %rcx,TF_RCX(%rsp) + movq %rsi,TF_RSI(%rsp) + movq %r8,TF_R8(%rsp) + movq %r9,TF_R9(%rsp) + movq %rbx,TF_RBX(%rsp) + movq %rbp,TF_RBP(%rsp) + movq %r10,TF_R10(%rsp) + movq %r11,TF_R11(%rsp) + movq %r12,TF_R12(%rsp) + movq %r13,TF_R13(%rsp) + movq %r14,TF_R14(%rsp) + movq %r15,TF_R15(%rsp) + SAVE_SEGS + movl $(TF_FRED_EVINFO2_TYPE_EXC | IDT_GP), TF_FRED_EVINFO2(%rsp) + movq %rsp, %rdi + KMSAN_ENTER + call trap_fred_u + KMSAN_LEAVE + jmp fred_checkast + +/* + * Content for the FRED entries page. +*/ + + .p2align 12,0x90 + .globl fred_events_handlers, fred_events_handlers_end +fred_events_handlers: + subq $TF_ERR,%rsp + movq %rax,TF_RAX(%rsp) + movq %rdi,TF_RDI(%rsp) + movq %rsi,TF_RSI(%rsp) + movl TF_FRED_EVINFO2(%rsp),%edi + movl %edi,%eax + andl $TF_FRED_EVINFO2_TYPEMASK,%edi + cmpl $TF_FRED_EVINFO2_TYPE_SYSCALL,%edi + je user_fred_syscall + jmp user_fred_non_syscall + + . = fred_events_handlers + 256 + subq $TF_ERR,%rsp + movq %rax,TF_RAX(%rsp) + movq %rdi,TF_RDI(%rsp) + movq %rdx,TF_RDX(%rsp) + movq %rcx,TF_RCX(%rsp) + movq %rsi,TF_RSI(%rsp) + movq %r8,TF_R8(%rsp) + movq %r9,TF_R9(%rsp) + movq %rbx,TF_RBX(%rsp) + movq %rbp,TF_RBP(%rsp) + movq %r10,TF_R10(%rsp) + movq %r11,TF_R11(%rsp) + movq %r12,TF_R12(%rsp) + movq %r13,TF_R13(%rsp) + movq %r14,TF_R14(%rsp) + movq %r15,TF_R15(%rsp) + movq %rsp, %rdi + KMSAN_ENTER + call trap_fred_k + KMSAN_LEAVE + RESTORE_REGS + addq $TF_ERR,%rsp + erets +fred_events_handlers_end: + + .globl fred_out_of_nmi +fred_out_of_nmi: + pushq %rbp + movq %rsp,%rbp + xorl %eax,%eax + pushq $0 + pushq $0 + movw %ss,%ax + orl $(1<<18),%eax /* NMI block release for ERETS */ + pushq %rax + pushq %rbp + pushfq + xorl %eax,%eax + movw %cs,%ax + pushq %rax + pushq $1f + pushq $0 + erets +1: popq %rbp + retq + #ifdef HWPMC_HOOKS ENTRY(end_exceptions) #endif diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 875c536166d1..9a5a74b3f6d6 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -146,6 +146,12 @@ ASSYM(PCB_STAR, offsetof(struct pcb, pcb_star)); ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar)); ASSYM(PCB_CSTAR, offsetof(struct pcb, pcb_cstar)); ASSYM(PCB_SFMASK, offsetof(struct pcb, pcb_sfmask)); +ASSYM(PCB_FRED_RSP0, offsetof(struct pcb, pcb_fred_rsp0)); +ASSYM(PCB_FRED_RSP1, offsetof(struct pcb, pcb_fred_rsp1)); +ASSYM(PCB_FRED_RSP2, offsetof(struct pcb, pcb_fred_rsp2)); +ASSYM(PCB_FRED_RSP3, offsetof(struct pcb, pcb_fred_rsp3)); +ASSYM(PCB_FRED_STKLVLS, offsetof(struct pcb, pcb_fred_stklvls)); +ASSYM(PCB_FRED_CONFIG, offsetof(struct pcb, pcb_fred_config)); ASSYM(PCB_SIZE, sizeof(struct pcb)); ASSYM(PCB_FULL_IRET, PCB_FULL_IRET); ASSYM(PCB_DBREGS, PCB_DBREGS); @@ -180,9 +186,15 @@ ASSYM(TF_DS, offsetof(struct trapframe, tf_ds)); ASSYM(TF_ES, offsetof(struct trapframe, tf_es)); ASSYM(TF_FS, offsetof(struct trapframe, tf_fs)); ASSYM(TF_GS, offsetof(struct trapframe, tf_gs)); +ASSYM(TF_FRED_EVINFO2, offsetof(struct trapframe, tf_fred_evinfo2)); ASSYM(TF_FLAGS, offsetof(struct trapframe, tf_flags)); ASSYM(TF_SIZE, sizeof(struct trapframe)); ASSYM(TF_HASSEGS, TF_HASSEGS); +ASSYM(TF_FRED_EVINFO2_TYPE_EXC, TF_FRED_EVINFO2_TYPE_EXC); +ASSYM(TF_FRED_EVINFO2_TYPE_SYSCALL, TF_FRED_EVINFO2_TYPE_SYSCALL); +ASSYM(TF_FRED_EVINFO2_TYPEMASK, TF_FRED_EVINFO2_TYPEMASK); +ASSYM(TF_FRED_EVINFO2_INSTLENMASK, TF_FRED_EVINFO2_INSTLENMASK); +ASSYM(TF_FRED_EVINFO2_INSTLENSHIFT, TF_FRED_EVINFO2_INSTLENSHIFT); ASSYM(PTI_RDX, offsetof(struct pti_frame, pti_rdx)); ASSYM(PTI_RAX, offsetof(struct pti_frame, pti_rax)); @@ -272,6 +284,9 @@ ASSYM(TSSSEL, GSEL(GPROC0_SEL, SEL_KPL)); ASSYM(LDTSEL, GSEL(GUSERLDT_SEL, SEL_KPL)); ASSYM(SEL_RPL_MASK, SEL_RPL_MASK); +ASSYM(IDT_NP, IDT_NP); +ASSYM(IDT_GP, IDT_GP); + ASSYM(__FreeBSD_version, __FreeBSD_version); #ifdef HWPMC_HOOKS diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c index 7161cf631508..5b615be4b8dc 100644 --- a/sys/amd64/amd64/initcpu.c +++ b/sys/amd64/amd64/initcpu.c @@ -397,3 +397,46 @@ initializecpucache(void) cpu_stdext_feature &= ~CPUID_STDEXT_CLFLUSHOPT; } } + +extern const char fred_events_handlers[], fred_events_handlers_end[]; + +void +amd64_cpu_init_fred(void) +{ + MPASS((((uintptr_t)fred_events_handlers) & PAGE_MASK) == 0); + /* + * 128b = 0x2 * 64b redzone + * SLC 0 for EXTINT + * SLC 0 current + */ + wrmsr(MSR_FRED_CONFIG, (uintptr_t)fred_events_handlers | + (0x2 << IA32_FRED_CONFIG_REDZONESZ_SHIFT) | + (0 << IA32_FRED_CONFIG_EXTINT_SLC_SHIFT) | 0); + + /* + * With FRED, there is no need for workarounds requiring + * dedicated stack for #DB. Also, there is no need to have + * separate stacks for #NM and #MC, because CSL rules ensure + * that interrupted frame is safe even if NMI was interrupted + * by machine check or vice versa. + * + * Still, we need to provide a stack for events that can occur + * with the interrupts disabled, to avoid caring about the + * ordering of the context update in cpu_switch.S. Use CSL 1 + * for #NM and #MC. + * + * Since double fault handler is used to handle stack + * overflow, it gets the dedicated stack, of the highest used + * CSL 2. + * + * CSL 3 is unused, and we have some space to re-arrange this + * if a need in the future appears. + */ + wrmsr(MSR_FRED_STKLVLS, (1ul << (IDT_NM * 2)) | + (1ul << (IDT_MC * 2)) | (2ul << (IDT_DF * 2))); + wrmsr(MSR_FRED_RSP3, 0); + load_cr4(rcr4() | CR4_FRED); + if (IS_BSP() && bootverbose) + printf("FRED enabled\n"); + +} diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index ab9edfe2c68f..c6a77c5ef852 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -206,10 +206,14 @@ long realmem = 0; int late_console = 1; int lass_enabled = 0; +int __read_frequently fred = 0; +SYSCTL_INT(_hw, OID_AUTO, fred, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, + &fred, 0, + "FRED is used"); + struct kva_md_info kmi; struct region_descriptor r_idt; - struct pcpu *__pcpu; struct pcpu temp_bsp_pcpu; @@ -347,9 +351,9 @@ cpu_setregs(void) static struct gate_descriptor idt0[NIDT]; struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */ -static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(16); +static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(64); static char mce0_stack[MCE_STACK_SIZE] __aligned(16); -static char nmi0_stack[NMI_STACK_SIZE] __aligned(16); +static char nmi0_stack[NMI_STACK_SIZE] __aligned(64); static char dbg0_stack[DBG_STACK_SIZE] __aligned(16); CTASSERT(sizeof(struct nmi_pcpu) == 16); @@ -485,6 +489,9 @@ setidt(int idx, inthand_t *func, int typ, int dpl, int ist) { struct gate_descriptor *ip; + if (fred) + return; + ip = idt + idx; ip->gd_looffset = (uintptr_t)func; ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL); @@ -1186,9 +1193,11 @@ amd64_conf_fast_syscall(void) msr = rdmsr(MSR_EFER) | EFER_SCE; wrmsr(MSR_EFER, msr); - wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) : - (u_int64_t)IDTVEC(fast_syscall)); - wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); + if (!fred) { + wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) : + (u_int64_t)IDTVEC(fast_syscall)); + wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); + } msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48); wrmsr(MSR_STAR, msr); @@ -1230,33 +1239,49 @@ amd64_bsp_ist_init(struct pcpu *pc) tssp = &pc->pc_common_tss; - /* doublefault stack space, runs on ist1 */ - np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(dblfault_stack)]) - 1; - np->np_pcpu = (register_t)pc; - tssp->tss_ist1 = (long)np; + /* Doublefault stack space, runs on ist1 for IDT. */ + if (fred) { + wrmsr(MSR_FRED_RSP2, (uint64_t)&dblfault_stack[ + sizeof(dblfault_stack)]); + } else { + np = ((struct nmi_pcpu *)&dblfault_stack[sizeof( + dblfault_stack)]) - 1; + np->np_pcpu = (register_t)pc; + tssp->tss_ist1 = (long)np; + } /* - * NMI stack, runs on ist2. The pcpu pointer is stored just - * above the start of the ist2 stack. + * NMI stack. */ - np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1; - np->np_pcpu = (register_t)pc; - tssp->tss_ist2 = (long)np; + if (fred) { + wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi0_stack[ + sizeof(nmi0_stack)]); + } else { + /* + * Runs on ist2 for IDT. The pcpu pointer is stored + * just above the start of the ist2 stack. + */ + np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1; + np->np_pcpu = (register_t)pc; + tssp->tss_ist2 = (long)np; + } - /* - * MC# stack, runs on ist3. The pcpu pointer is stored just - * above the start of the ist3 stack. - */ - np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1; - np->np_pcpu = (register_t)pc; - tssp->tss_ist3 = (long)np; + if (!fred) { + /* + * MC# stack for IDT, runs on ist3. The pcpu pointer + * is stored just above the start of the ist3 stack. + */ + np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1; + np->np_pcpu = (register_t)pc; + tssp->tss_ist3 = (long)np; - /* - * DB# stack, runs on ist4. - */ - np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1; - np->np_pcpu = (register_t)pc; - tssp->tss_ist4 = (long)np; + /* + * DB# stack for IDT, runs on ist4. + */ + np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1; + np->np_pcpu = (register_t)pc; + tssp->tss_ist4 = (long)np; + } } /* @@ -1359,6 +1384,13 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); } + if ((cpu_stdext_feature4 & (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS)) == + (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS) && + (cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 && !pti) { + fred = 1; + TUNABLE_INT_FETCH("hw.fred", &fred); + } + sched_instance_select(); link_elf_ireloc(); @@ -1473,9 +1505,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) : &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0); #endif - r_idt.rd_limit = sizeof(idt0) - 1; - r_idt.rd_base = (long) idt; - lidt(&r_idt); + if (!fred) { + r_idt.rd_limit = sizeof(idt0) - 1; + r_idt.rd_base = (long) idt; + lidt(&r_idt); + } TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable); TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable); @@ -1564,6 +1598,9 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) if (getenv_is_true("debug.dump_modinfo_at_boot")) preload_dump(); + if (fred) + amd64_cpu_init_fred(); + #ifdef DEV_ISA #ifdef DEV_ATPIC elcr_probe(); @@ -1828,6 +1865,11 @@ clear_pcb_flags(struct pcb *pcb, const u_int flags) extern const char wrmsr_early_safe_gp_handler[]; +/* + * What about FRED? wrmsr_early_safe_start() is used before we + * switched CPU to the FRED mode. We use IDT to catch #GP from MSR + * write even if BSP is switched to the FRED mode later. + */ void wrmsr_early_safe_start(void) { diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 69490b8e275f..81a24a4f4df5 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -240,25 +240,51 @@ init_secondary(void) IOPERM_BITMAP_SIZE; pc->pc_common_tss.tss_rsp0 = 0; - /* The doublefault stack runs on IST1. */ - np = ((struct nmi_pcpu *)&doublefault_stack[DBLFAULT_STACK_SIZE]) - 1; - np->np_pcpu = (register_t)pc; - pc->pc_common_tss.tss_ist1 = (long)np; - - /* The NMI stack runs on IST2. */ - np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1; - np->np_pcpu = (register_t)pc; - pc->pc_common_tss.tss_ist2 = (long)np; - - /* The MC# stack runs on IST3. */ - np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1; - np->np_pcpu = (register_t)pc; - pc->pc_common_tss.tss_ist3 = (long)np; - - /* The DB# stack runs on IST4. */ - np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1; - np->np_pcpu = (register_t)pc; - pc->pc_common_tss.tss_ist4 = (long)np; + /* + * The doublefault stack. + * Runs on IST1 for IDT. + * Uses CSL 2 for FRED. + */ + if (fred) { + wrmsr(MSR_FRED_RSP2, (uint64_t)&doublefault_stack[ + DBLFAULT_STACK_SIZE]); + } else { + np = ((struct nmi_pcpu *)&doublefault_stack[ + DBLFAULT_STACK_SIZE]) - 1; + np->np_pcpu = (register_t)pc; + pc->pc_common_tss.tss_ist1 = (long)np; + } + + /* + * The NMI stack. + * Runs on IST2 for IDT. + * Uses CSL 1 for FRED. + */ + if (fred) { + wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi_stack[NMI_STACK_SIZE]); + } else { + np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1; + np->np_pcpu = (register_t)pc; + pc->pc_common_tss.tss_ist2 = (long)np; + } + + /* + * The MC# stack. + * Runs on IST3 for IDT. + * Shares CSL 1 with NMI for FRED. + */ + if (!fred) { + np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1; + np->np_pcpu = (register_t)pc; + pc->pc_common_tss.tss_ist3 = (long)np; + } + + if (!fred) { + /* The DB# stack, used for for IDT, runs on IST4. */ + np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1; + np->np_pcpu = (register_t)pc; + pc->pc_common_tss.tss_ist4 = (long)np; + } /* Prepare private GDT */ gdt_segs[GPROC0_SEL].ssd_base = (long)&pc->pc_common_tss; @@ -278,7 +304,10 @@ init_secondary(void) wrmsr(MSR_KGSBASE, 0); /* User value */ fix_cpuid(); - lidt(&r_idt); + if (fred) + amd64_cpu_init_fred(); + else + lidt(&r_idt); gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); ltr(gsel_tss); @@ -430,12 +459,17 @@ start_all_aps(void) M_WAITOK | M_ZERO); doublefault_stack = kmem_malloc(DBLFAULT_STACK_SIZE, M_WAITOK | M_ZERO); - mce_stack = kmem_malloc(MCE_STACK_SIZE, - M_WAITOK | M_ZERO); + if (!fred) { + mce_stack = kmem_malloc(MCE_STACK_SIZE, + M_WAITOK | M_ZERO); + } nmi_stack = kmem_malloc_domainset( DOMAINSET_PREF(domain), NMI_STACK_SIZE, M_WAITOK | M_ZERO); - dbg_stack = kmem_malloc_domainset( - DOMAINSET_PREF(domain), DBG_STACK_SIZE, M_WAITOK | M_ZERO); + if (!fred) { + dbg_stack = kmem_malloc_domainset( + DOMAINSET_PREF(domain), DBG_STACK_SIZE, + M_WAITOK | M_ZERO); + } dpcpu = kmem_malloc_domainset(DOMAINSET_PREF(domain), DPCPU_SIZE, M_WAITOK | M_ZERO); diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index df9cc44bcbb2..815d356217f9 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -41,6 +41,7 @@ * AMD64 Trap and System call handling */ +#include "opt_atpic.h" #include "opt_clock.h" #include "opt_cpu.h" #include "opt_hwpmc_hooks.h" @@ -61,6 +62,7 @@ #include <sys/mutex.h> #include <sys/resourcevar.h> #include <sys/signalvar.h> +#include <sys/smp.h> #include <sys/syscall.h> #include <sys/sysctl.h> #include <sys/sysent.h> @@ -83,6 +85,8 @@ PMC_SOFT_DEFINE( , , page_fault, write); #include <machine/cpu.h> #include <machine/intr_machdep.h> +#include <x86/apicreg.h> +#include <x86/apicvar.h> #include <x86/mca.h> #include <machine/md_var.h> #include <machine/pcb.h> @@ -90,6 +94,7 @@ PMC_SOFT_DEFINE( , , page_fault, write); #include <machine/stack.h> #include <machine/trap.h> #include <machine/tss.h> +#include <x86/isa/icu.h> #ifdef KDTRACE_HOOKS #include <sys/dtrace_bsd.h> @@ -995,6 +1000,14 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str) frame->tf_r11, frame->tf_r12); printf("r13: %016lx r14: %016lx r15: %016lx\n", frame->tf_r13, frame->tf_r14, frame->tf_r15); + if (fred) { + struct trapframe_fred *f; + + f = __containerof(frame, struct trapframe_fred, tf_idt); + printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n", + f->tf_fred_evdata, frame->tf_fred_evinfo1, + frame->tf_fred_evinfo2); + } } static void @@ -1069,6 +1082,14 @@ dblfault_handler(struct trapframe *frame) frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es, frame->tf_fs, frame->tf_gs, rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE)); + if (fred) { + struct trapframe_fred *f; + + f = __containerof(frame, struct trapframe_fred, tf_idt); + printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n", + f->tf_fred_evdata, frame->tf_fred_evinfo1, + frame->tf_fred_evinfo2); + } /* Print these separately in case pcpu accesses trap. */ printf("cpuid = %d; apic id = %02x\n", PCPU_GET(cpuid), PCPU_GET(apic_id)); @@ -1295,3 +1316,540 @@ amd64_syscall(struct thread *td, int traced) amd64_syscall_ret_flush_l1d_check_inline(td->td_errno); } + +static void +trap_fred_extint(struct trapframe_fred *frame, u_int vec) +{ + switch (vec) { + case IPI_RENDEZVOUS: +#ifdef COUNT_IPIS + ipi_rendezvous_counts[PCPU_GET(cpuid)]++; +#endif + smp_rendezvous_action(); + lapic_eoi(); + break; + case IPI_INVLOP: + invlop_handler(); + lapic_eoi(); + break; + case IPI_BITMAP_VECTOR: + lapic_eoi(); + ipi_bitmap_handler(&frame->tf_idt); + break; + case IPI_STOP: + lapic_eoi(); + cpustop_handler(); + break; + case IPI_SUSPEND: + cpususpend_handler(); + lapic_eoi(); + break; + case IPI_SWI: + lapic_eoi(); + ipi_swi_handler(&frame->tf_idt); + break; + case IPI_OFF: + cpuoff_handler(); + lapic_eoi(); + break; + case APIC_SPURIOUS_INT: + break; + case APIC_TIMER_INT: + lapic_handle_timer(&frame->tf_idt); + break; + case APIC_CMC_INT: + lapic_handle_cmc(); + break; + case APIC_ERROR_INT: + lapic_handle_error(); + break; + case APIC_THERMAL_INT: + lapic_handle_thermal(); + break; +#ifdef XENHVM + case IDT_EVTCHN: + xen_arch_intr_handle_upcall(&frame->tf_idt); + break; +#endif + default: + if (vec >= IPI_DYN_FIRST && vec <= IPI_DYN_LAST) { + void (*fi)(struct trapframe *); + + fi = fred_ipi_handlers[vec - IPI_DYN_FIRST]; + if (fi != NULL) { *** 749 LINES SKIPPED ***
