The branch main has been updated by jhibbits: URL: https://cgit.FreeBSD.org/src/commit/?id=de5fcd16650207f6f2e7c50afd1ebbaeb36612c7
commit de5fcd16650207f6f2e7c50afd1ebbaeb36612c7 Author: Justin Hibbits <[email protected]> AuthorDate: 2026-09-05 18:38:55 +0000 Commit: Justin Hibbits <[email protected]> CommitDate: 2026-09-12 02:40:30 +0000 powerpc/booke: Add a machine check stack Machine check and critical exceptions are independent of each other, and can interrupt each other. Since they're asynchronous we cannot trust that the existing stack pointer (%r1) is correct at time of entry, so add a private machine check stack separate from the critical exception stack. As part of this, switch to the STANDARD_CRIT_PROLOG() for machine check exceptions, and overload the macro to specify the stack to switch to. Also add a savearea argument to CRIT_SRR_RESTORE() so that we can restore machine check exception state from the right location. --- sys/powerpc/booke/booke_machdep.c | 14 ++++++++++++-- sys/powerpc/booke/trap_subr.S | 30 ++++++++++++++++++------------ sys/powerpc/include/pcpu.h | 7 ++++--- sys/powerpc/powerpc/genassym.c | 1 + 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/sys/powerpc/booke/booke_machdep.c b/sys/powerpc/booke/booke_machdep.c index 58b847766d46..21b9852945d6 100644 --- a/sys/powerpc/booke/booke_machdep.c +++ b/sys/powerpc/booke/booke_machdep.c @@ -398,22 +398,32 @@ extern uintptr_t tlb0_miss_locks[]; * MAXCPU array here would reserve megabytes for CPUs that do not exist. */ static char booke_boot_critstack[BOOKE_CRITSTACK_SIZE] __aligned(16); +static char booke_boot_mchkstack[BOOKE_CRITSTACK_SIZE] __aligned(16); static bool booke_boot_critstack_used; /* Initialise a struct pcpu. */ void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz) { - char *critstack; + char *critstack, *mchkstack; pcpu->pc_booke.tid_next = TID_MIN; + /* + * Machine check and critical interrupts have their own stacks with + * their own stack pointer, because regular mode registers cannot be + * trusted. + */ if (!booke_boot_critstack_used) { booke_boot_critstack_used = true; critstack = booke_boot_critstack; - } else + mchkstack = booke_boot_mchkstack; + } else { critstack = malloc(BOOKE_CRITSTACK_SIZE, M_DEVBUF, M_WAITOK); + mchkstack = malloc(BOOKE_CRITSTACK_SIZE, M_DEVBUF, M_WAITOK); + } pcpu->pc_booke.critstack = critstack + BOOKE_CRITSTACK_SIZE; + pcpu->pc_booke.mchkstack = mchkstack + BOOKE_CRITSTACK_SIZE; #ifdef SMP uintptr_t *ptr; diff --git a/sys/powerpc/booke/trap_subr.S b/sys/powerpc/booke/trap_subr.S index 55e4d03639d4..69ed40cdd8ca 100644 --- a/sys/powerpc/booke/trap_subr.S +++ b/sys/powerpc/booke/trap_subr.S @@ -136,7 +136,7 @@ LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1: -#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ +#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, stack, isrr0, isrr1) \ mtspr sprg_sp, %r1; /* Save SP */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ @@ -166,7 +166,7 @@ bf 17, 1f; \ LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ b 2f; \ -1: LOAD %r1, PC_BOOKE_CRITSTACK(%r1); \ +1: LOAD %r1, stack(%r1); \ 2: /* @@ -177,10 +177,10 @@ * a TLB miss return. Nested exceptions taken by the C dispatcher clobber * them either way, so restore before returning. */ -#define CRIT_SRR_RESTORE \ +#define CRIT_SRR_RESTORE(savearea) \ GET_CPUINFO(%r3); \ - LOAD %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r3); \ - LOAD %r5, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r3); \ + LOAD %r4, (savearea+BOOKE_CRITSAVE_SRR0)(%r3); \ + LOAD %r5, (savearea+BOOKE_CRITSAVE_SRR1)(%r3); \ mtspr SPR_SRR0, %r4; \ mtspr SPR_SRR1, %r5 @@ -554,13 +554,14 @@ INTERRUPT(int_unknown) * Critical input interrupt ****************************************************************************/ INTERRUPT(int_critical_input) - STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) + STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, + PC_BOOKE_CRITSTACK, SPR_CSRR0, SPR_CSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) GET_TOCBASE(%r2) addi %r3, %r1, CALLSIZE bl CNAME(powerpc_interrupt) TOC_RESTORE - CRIT_SRR_RESTORE + CRIT_SRR_RESTORE(PC_BOOKE_CRITSAVE) FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) rfci @@ -569,12 +570,14 @@ INTERRUPT(int_critical_input) * Machine check interrupt ****************************************************************************/ INTERRUPT(int_machine_check) - STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) + STANDARD_CRIT_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, PC_BOOKE_MCHKSTACK, + SPR_MCSRR0, SPR_MCSRR1) FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) GET_TOCBASE(%r2) addi %r3, %r1, CALLSIZE bl CNAME(powerpc_interrupt) TOC_RESTORE + CRIT_SRR_RESTORE(PC_BOOKE_MCHKSAVE) FRAME_LEAVE(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) rfmci @@ -655,13 +658,14 @@ INTERRUPT(int_fixed_interval_timer) * Watchdog interrupt ****************************************************************************/ INTERRUPT(int_watchdog) - STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) + STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, + PC_BOOKE_CRITSTACK, SPR_CSRR0, SPR_CSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_WDOG) GET_TOCBASE(%r2) addi %r3, %r1, CALLSIZE bl CNAME(powerpc_interrupt) TOC_RESTORE - CRIT_SRR_RESTORE + CRIT_SRR_RESTORE(PC_BOOKE_CRITSAVE) FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) rfci @@ -981,14 +985,16 @@ interrupt_vector_top: * Debug interrupt ****************************************************************************/ INTERRUPT(int_debug) - STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) + STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, + PC_BOOKE_CRITSTACK, SPR_CSRR0, SPR_CSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) bl int_debug_int FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) rfci INTERRUPT(int_debug_ed) - STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1) + STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, + PC_BOOKE_CRITSTACK, SPR_DSRR0, SPR_DSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) bl int_debug_int FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1) diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h index 18f0e3209b92..752e34639640 100644 --- a/sys/powerpc/include/pcpu.h +++ b/sys/powerpc/include/pcpu.h @@ -93,17 +93,18 @@ struct pvo_entry; #define BOOKE_CRITSTACK_SIZE 16384 #ifdef __powerpc64__ -#define BOOKE_PCPU_PAD 893 +#define BOOKE_PCPU_PAD 869 #else -#define BOOKE_PCPU_PAD 361 +#define BOOKE_PCPU_PAD 349 #endif #define PCPU_MD_BOOKE_FIELDS \ register_t critsave[BOOKE_CRITSAVE_LEN]; \ - register_t mchksave[CPUSAVE_LEN]; \ + register_t mchksave[BOOKE_CRITSAVE_LEN]; \ register_t tlbsave[BOOKE_TLBSAVE_LEN]; \ register_t tlb_level; \ uintptr_t *tlb_lock; \ void *critstack; \ + void *mchkstack; \ int tid_next; \ char __pad[BOOKE_PCPU_PAD]; diff --git a/sys/powerpc/powerpc/genassym.c b/sys/powerpc/powerpc/genassym.c index e8c08867d778..4105575b11ee 100644 --- a/sys/powerpc/powerpc/genassym.c +++ b/sys/powerpc/powerpc/genassym.c @@ -70,6 +70,7 @@ ASSYM(PC_BOOKE_TLBSAVE, offsetof(struct pcpu, pc_booke.tlbsave)); ASSYM(PC_BOOKE_TLB_LEVEL, offsetof(struct pcpu, pc_booke.tlb_level)); ASSYM(PC_BOOKE_TLB_LOCK, offsetof(struct pcpu, pc_booke.tlb_lock)); ASSYM(PC_BOOKE_CRITSTACK, offsetof(struct pcpu, pc_booke.critstack)); +ASSYM(PC_BOOKE_MCHKSTACK, offsetof(struct pcpu, pc_booke.mchkstack)); #endif ASSYM(CPUSAVE_R27, CPUSAVE_R27*sizeof(register_t));
