The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=9e0143694ad31b534310a473b993e520264f11e2
commit 9e0143694ad31b534310a473b993e520264f11e2 Author: Konstantin Belousov <[email protected]> AuthorDate: 2023-05-01 11:02:59 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2023-05-02 11:32:35 +0000 amd64: add pmap_get_pcid() helper Reviewed by: markj Tested by: pho Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D39890 --- sys/amd64/amd64/efirt_machdep.c | 19 +++++++++++++------ sys/amd64/amd64/mp_machdep.c | 20 ++++++++------------ sys/amd64/amd64/pmap.c | 16 ++++------------ sys/amd64/include/pmap.h | 8 ++++++++ 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/sys/amd64/amd64/efirt_machdep.c b/sys/amd64/amd64/efirt_machdep.c index 6fffd113f868..b7f7a91b8671 100644 --- a/sys/amd64/amd64/efirt_machdep.c +++ b/sys/amd64/amd64/efirt_machdep.c @@ -298,6 +298,7 @@ int efi_arch_enter(void) { pmap_t curpmap; + uint64_t cr3; curpmap = PCPU_GET(curpmap); PMAP_LOCK_ASSERT(curpmap, MA_OWNED); @@ -313,8 +314,10 @@ efi_arch_enter(void) if (pmap_pcid_enabled && !invpcid_works) PCPU_SET(curpmap, NULL); - load_cr3(VM_PAGE_TO_PHYS(efi_pmltop_page) | (pmap_pcid_enabled ? - curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0)); + cr3 = VM_PAGE_TO_PHYS(efi_pmltop_page); + if (pmap_pcid_enabled) + cr3 |= pmap_get_pcid(curpmap); + load_cr3(cr3); /* * If PCID is enabled, the clear CR3_PCID_SAVE bit in the loaded %cr3 * causes TLB invalidation. @@ -328,12 +331,16 @@ void efi_arch_leave(void) { pmap_t curpmap; + uint64_t cr3; curpmap = &curproc->p_vmspace->vm_pmap; - if (pmap_pcid_enabled && !invpcid_works) - PCPU_SET(curpmap, curpmap); - load_cr3(curpmap->pm_cr3 | (pmap_pcid_enabled ? - curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0)); + cr3 = curpmap->pm_cr3; + if (pmap_pcid_enabled) { + cr3 |= pmap_get_pcid(curpmap); + if (!invpcid_works) + PCPU_SET(curpmap, curpmap); + } + load_cr3(cr3); if (!pmap_pcid_enabled) invltlb(); vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf); diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 5c60d301c1e7..94589f736637 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -767,7 +767,7 @@ invltlb_invpcid_handler(pmap_t smp_tlb_pmap) (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; + d.pcid = pmap_get_pcid(smp_tlb_pmap); d.pad = 0; d.addr = 0; invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB : @@ -786,7 +786,7 @@ invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap) (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; + d.pcid = pmap_get_pcid(smp_tlb_pmap); d.pad = 0; d.addr = 0; if (smp_tlb_pmap == kernel_pmap) { @@ -808,8 +808,6 @@ invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap) static void invltlb_pcid_handler(pmap_t smp_tlb_pmap) { - uint32_t pcid; - #ifdef COUNT_XINVLTLB_HITS xhits_gbl[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -828,8 +826,8 @@ invltlb_pcid_handler(pmap_t smp_tlb_pmap) * CPU. */ if (smp_tlb_pmap == PCPU_GET(curpmap)) { - pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; - load_cr3(smp_tlb_pmap->pm_cr3 | pcid); + load_cr3(smp_tlb_pmap->pm_cr3 | + pmap_get_pcid(smp_tlb_pmap)); if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3) PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE); } @@ -865,8 +863,7 @@ invlpg_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1) if (smp_tlb_pmap == PCPU_GET(curpmap) && smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 && PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { - d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid | - PMAP_PCID_USER_PT; + d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT; d.pad = 0; d.addr = smp_tlb_addr1; invpcid(&d, INVPCID_ADDR); @@ -890,7 +887,7 @@ invlpg_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1) if (smp_tlb_pmap == PCPU_GET(curpmap) && (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 && PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { - pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; + pcid = pmap_get_pcid(smp_tlb_pmap); kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1); @@ -944,8 +941,7 @@ invlrng_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1, if (smp_tlb_pmap == PCPU_GET(curpmap) && smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 && PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { - d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid | - PMAP_PCID_USER_PT; + d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT; d.pad = 0; d.addr = smp_tlb_addr1; do { @@ -978,7 +974,7 @@ invlrng_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1, if (smp_tlb_pmap == PCPU_GET(curpmap) && (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 && PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { - pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; + pcid = pmap_get_pcid(smp_tlb_pmap); kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2); diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 3b125041bb29..16bfe9986c6b 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -3076,7 +3076,6 @@ pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va, struct invpcid_descr d; uint64_t kcr3, ucr3; uint32_t pcid; - u_int cpuid; /* * Because pm_pcid is recalculated on a context switch, we @@ -3095,9 +3094,7 @@ pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va, PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK) return; - cpuid = PCPU_GET(cpuid); - - pcid = pmap->pm_pcids[cpuid].pm_pcid; + pcid = pmap_get_pcid(pmap); if (invpcid_works1) { d.pcid = pcid | PMAP_PCID_USER_PT; d.pad = 0; @@ -3172,7 +3169,6 @@ pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, struct invpcid_descr d; uint64_t kcr3, ucr3; uint32_t pcid; - u_int cpuid; CRITICAL_ASSERT(curthread); @@ -3181,9 +3177,7 @@ pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK) return; - cpuid = PCPU_GET(cpuid); - - pcid = pmap->pm_pcids[cpuid].pm_pcid; + pcid = pmap_get_pcid(pmap); if (invpcid_works1) { d.pcid = pcid | PMAP_PCID_USER_PT; d.pad = 0; @@ -3273,7 +3267,6 @@ pmap_invalidate_all_pcid_cb(pmap_t pmap, bool invpcid_works1) struct invpcid_descr d; uint64_t kcr3; uint32_t pcid; - u_int cpuid; if (pmap == kernel_pmap) { if (invpcid_works1) { @@ -3284,9 +3277,8 @@ pmap_invalidate_all_pcid_cb(pmap_t pmap, bool invpcid_works1) } } else if (pmap == PCPU_GET(curpmap)) { CRITICAL_ASSERT(curthread); - cpuid = PCPU_GET(cpuid); - pcid = pmap->pm_pcids[cpuid].pm_pcid; + pcid = pmap_get_pcid(pmap); if (invpcid_works1) { d.pcid = pcid; d.pad = 0; @@ -10152,7 +10144,7 @@ pmap_activate_boot(pmap_t pmap) if (pti) { kcr3 = pmap->pm_cr3; if (pmap_pcid_enabled) - kcr3 |= pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE; + kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE; } else { kcr3 = PMAP_NO_CR3; } diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 3b99b88742e5..cef718fb50d5 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -533,6 +533,14 @@ pmap_invlpg(pmap_t pmap, vm_offset_t va) } #endif /* sys/pcpu.h && machine/cpufunc.h */ +/* Return pcid for the pmap pmap on current cpu */ +static __inline uint32_t +pmap_get_pcid(pmap_t pmap) +{ + MPASS(pmap_pcid_enabled); + return (pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid); +} + #endif /* _KERNEL */ /* Return various clipped indexes for a given VA */
