The branch main has been updated by andrew: URL: https://cgit.FreeBSD.org/src/commit/?id=6419b48f7d7f3d9048c59945d4ba9ed1b59bb87c
commit 6419b48f7d7f3d9048c59945d4ba9ed1b59bb87c Author: Andrew Turner <[email protected]> AuthorDate: 2022-11-03 16:01:37 +0000 Commit: Andrew Turner <[email protected]> CommitDate: 2023-03-15 11:34:32 +0000 Support arm64 stage2 TLB invalidation To invalidate stage 2 mappings on arm64 we may need to call into the hypervisor so add a function pointer that bhyve can use to implement this. Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D37254 --- sys/arm64/arm64/pmap.c | 83 ++++++++++++++++++++++++++++++++++++++---------- sys/arm64/include/pmap.h | 3 ++ 2 files changed, 69 insertions(+), 17 deletions(-) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 357584e22875..a272bf5756ae 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -402,6 +402,8 @@ SYSCTL_INT(_vm_pmap_vmid, OID_AUTO, epoch, CTLFLAG_RD, &vmids.asid_epoch, 0, void (*pmap_clean_stage2_tlbi)(void); void (*pmap_invalidate_vpipt_icache)(void); +void (*pmap_stage2_invalidate_range)(uint64_t, vm_offset_t, vm_offset_t, bool); +void (*pmap_stage2_invalidate_all)(uint64_t); /* * A pmap's cookie encodes an ASID and epoch number. Cookies for reserved @@ -1549,6 +1551,24 @@ pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) isb(); } +static __inline void +pmap_s2_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) +{ + PMAP_ASSERT_STAGE2(pmap); + MPASS(pmap_stage2_invalidate_range != NULL); + pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), va, va + PAGE_SIZE, + final_only); +} + +static __inline void +pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) +{ + if (pmap->pm_stage == PM_STAGE1) + pmap_s1_invalidate_page(pmap, va, final_only); + else + pmap_s2_invalidate_page(pmap, va, final_only); +} + /* * Invalidates any cached final- and optionally intermediate-level TLB entries * for the specified virtual address range in the given virtual address space. @@ -1578,6 +1598,25 @@ pmap_s1_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, isb(); } +static __inline void +pmap_s2_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, + bool final_only) +{ + PMAP_ASSERT_STAGE2(pmap); + MPASS(pmap_stage2_invalidate_range != NULL); + pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), sva, eva, final_only); +} + +static __inline void +pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, + bool final_only) +{ + if (pmap->pm_stage == PM_STAGE1) + pmap_s1_invalidate_range(pmap, sva, eva, final_only); + else + pmap_s2_invalidate_range(pmap, sva, eva, final_only); +} + /* * Invalidates all cached intermediate- and final-level TLB entries for the * given virtual address space. @@ -1600,6 +1639,23 @@ pmap_s1_invalidate_all(pmap_t pmap) isb(); } +static __inline void +pmap_s2_invalidate_all(pmap_t pmap) +{ + PMAP_ASSERT_STAGE2(pmap); + MPASS(pmap_stage2_invalidate_all != NULL); + pmap_stage2_invalidate_all(pmap_to_ttbr0(pmap)); +} + +static __inline void +pmap_invalidate_all(pmap_t pmap) +{ + if (pmap->pm_stage == PM_STAGE1) + pmap_s1_invalidate_all(pmap); + else + pmap_s2_invalidate_all(pmap); +} + /* * Routine: pmap_extract * Function: @@ -2046,7 +2102,7 @@ _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); pmap_unwire_l3(pmap, va, l1pg, free); } - pmap_s1_invalidate_page(pmap, va, false); + pmap_invalidate_page(pmap, va, false); /* * Put page on a list so that it is released after @@ -3347,7 +3403,7 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva, for (l3 = pmap_l2_to_l3(&l2e, sva); sva != eva; l3++, sva += L3_SIZE) { if (!pmap_l3_valid(pmap_load(l3))) { if (va != eva) { - pmap_s1_invalidate_range(pmap, va, sva, true); + pmap_invalidate_range(pmap, va, sva, true); va = eva; } continue; @@ -3374,7 +3430,7 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva, * still provides access to that page. */ if (va != eva) { - pmap_s1_invalidate_range(pmap, va, + pmap_invalidate_range(pmap, va, sva, true); va = eva; } @@ -3405,7 +3461,7 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva, va = sva; } if (va != eva) - pmap_s1_invalidate_range(pmap, va, sva, true); + pmap_invalidate_range(pmap, va, sva, true); } /* @@ -4311,12 +4367,6 @@ havel3: * Is the specified virtual address already mapped? */ if (pmap_l3_valid(orig_l3)) { - /* - * Only allow adding new entries on stage 2 tables for now. - * This simplifies cache invalidation as we may need to call - * into EL2 to perform such actions. - */ - PMAP_ASSERT_STAGE1(pmap); /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there @@ -4371,7 +4421,7 @@ havel3: if (pmap_pte_dirty(pmap, orig_l3)) vm_page_dirty(om); if ((orig_l3 & ATTR_AF) != 0) { - pmap_s1_invalidate_page(pmap, va, true); + pmap_invalidate_page(pmap, va, true); vm_page_aflag_set(om, PGA_REFERENCED); } CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); @@ -4386,7 +4436,7 @@ havel3: } else { KASSERT((orig_l3 & ATTR_AF) != 0, ("pmap_enter: unmanaged mapping lacks ATTR_AF")); - pmap_s1_invalidate_page(pmap, va, true); + pmap_invalidate_page(pmap, va, true); } orig_l3 = 0; } else { @@ -4439,12 +4489,11 @@ validate: * Update the L3 entry */ if (pmap_l3_valid(orig_l3)) { - PMAP_ASSERT_STAGE1(pmap); KASSERT(opa == pa, ("pmap_enter: invalid update")); if ((orig_l3 & ~ATTR_AF) != (new_l3 & ~ATTR_AF)) { /* same PA, different attributes */ orig_l3 = pmap_load_store(l3, new_l3); - pmap_s1_invalidate_page(pmap, va, true); + pmap_invalidate_page(pmap, va, true); if ((orig_l3 & ATTR_SW_MANAGED) != 0 && pmap_pte_dirty(pmap, orig_l3)) vm_page_dirty(m); @@ -5588,7 +5637,7 @@ pmap_remove_pages(pmap_t pmap) } if (lock != NULL) rw_wunlock(lock); - pmap_s1_invalidate_all(pmap); + pmap_invalidate_all(pmap); free_pv_chunk_batch(free_chunks); PMAP_UNLOCK(pmap); vm_page_free_pages_toq(&free, true); @@ -5913,7 +5962,7 @@ retry: (uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 && (tpte & ATTR_SW_WIRED) == 0) { pmap_clear_bits(pte, ATTR_AF); - pmap_s1_invalidate_page(pmap, va, true); + pmap_invalidate_page(pmap, va, true); cleared++; } else not_cleared++; @@ -5954,7 +6003,7 @@ small_mappings: if ((tpte & ATTR_AF) != 0) { if ((tpte & ATTR_SW_WIRED) == 0) { pmap_clear_bits(pte, ATTR_AF); - pmap_s1_invalidate_page(pmap, pv->pv_va, true); + pmap_invalidate_page(pmap, pv->pv_va, true); cleared++; } else not_cleared++; diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h index f61725e21b7c..eb72f46668c3 100644 --- a/sys/arm64/include/pmap.h +++ b/sys/arm64/include/pmap.h @@ -172,6 +172,9 @@ struct pcb *pmap_switch(struct thread *); extern void (*pmap_clean_stage2_tlbi)(void); extern void (*pmap_invalidate_vpipt_icache)(void); +extern void (*pmap_stage2_invalidate_range)(uint64_t, vm_offset_t, vm_offset_t, + bool); +extern void (*pmap_stage2_invalidate_all)(uint64_t); static inline int pmap_vmspace_copy(pmap_t dst_pmap __unused, pmap_t src_pmap __unused)
