The branch main has been updated by andrew: URL: https://cgit.FreeBSD.org/src/commit/?id=56abdbc5f709fc0e18624b3b7586647459922a41
commit 56abdbc5f709fc0e18624b3b7586647459922a41 Author: Sarah Walker <[email protected]> AuthorDate: 2026-05-12 11:29:20 +0000 Commit: Andrew Turner <[email protected]> CommitDate: 2026-05-12 16:54:41 +0000 arm64: VM/PMAP changes for CCA guest support When in a realm: - Mappings with mode VM_MEMATTR_DEVICE and VM_MEMATTR_DEVICE_NP are unprotected - Imported busdma buffers in protected memory are always bounced - If EARLY_PRINTK is in use, the UART physical address must be in the unprotected address space Reviewed by: andrew Sponsored by: Arm Ltd Differential Revision: https://reviews.freebsd.org/D56599 --- sys/arm64/arm64/busdma_bounce.c | 68 ++++++++++++++++++++++++++++++++++++++--- sys/arm64/arm64/efirt_machdep.c | 10 +++++- sys/arm64/arm64/pmap.c | 40 ++++++++++++++++++++++++ sys/conf/files.arm64 | 1 + sys/vm/vm_page.c | 8 +++++ 5 files changed, 122 insertions(+), 5 deletions(-) diff --git a/sys/arm64/arm64/busdma_bounce.c b/sys/arm64/arm64/busdma_bounce.c index 74fa611e6d1a..db7e108812aa 100644 --- a/sys/arm64/arm64/busdma_bounce.c +++ b/sys/arm64/arm64/busdma_bounce.c @@ -36,6 +36,7 @@ #include <sys/domainset.h> #include <sys/malloc.h> #include <sys/bus.h> +#include <sys/busdma_bufalloc.h> #include <sys/interrupt.h> #include <sys/kernel.h> #include <sys/ktr.h> @@ -56,6 +57,7 @@ #include <machine/atomic.h> #include <machine/bus.h> #include <machine/md_var.h> +#include <machine/rsi.h> #include <arm64/include/bus_dma_impl.h> #define MAX_BPAGES 4096 @@ -120,6 +122,8 @@ static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags); +static busdma_bufalloc_t nonsecure_allocator; + static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); #define dmat_alignment(dmat) ((dmat)->common.alignment) @@ -215,6 +219,10 @@ might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, if (map && (map->flags & DMAMAP_FROM_DMAMEM) != 0) return (false); + /* Bounce if accessing secure memory */ + if (in_realm() && !(paddr & prot_ns_shared_pa)) + return (true); + if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) return (true); @@ -239,6 +247,10 @@ must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, addr_needs_bounce(dmat, paddr)) return (true); + /* Bounce if accessing secure memory */ + if (in_realm() && !(paddr & prot_ns_shared_pa)) + return (true); + return (false); } @@ -492,6 +504,7 @@ static int bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp) { + struct busdma_bufzone *bufzone; vm_memattr_t attr; int mflags; @@ -524,6 +537,9 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, else attr = VM_MEMATTR_DEFAULT; + if (in_realm()) + mflags |= M_UNPROTECTED; + /* * Create the map, but don't set the could bounce flag as * this allocation should never bounce; @@ -567,13 +583,16 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, * * In the meantime warn the user if malloc gets it wrong. */ - if (dmat->alloc_size <= PAGE_SIZE && + + bufzone = busdma_bufalloc_findzone(nonsecure_allocator, + dmat->alloc_size); + + if (bufzone && + dmat->alloc_size <= PAGE_SIZE && dmat->alloc_alignment <= PAGE_SIZE && dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && attr == VM_MEMATTR_DEFAULT) { - *vaddr = malloc_domainset_aligned(dmat->alloc_size, - dmat->alloc_alignment, M_DEVBUF, - DOMAINSET_PREF(dmat->common.domain), mflags); + *vaddr = uma_zalloc(bufzone->umazone, mflags); } else if (dmat->common.nsegments >= howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && dmat->alloc_alignment <= PAGE_SIZE && @@ -1148,3 +1167,44 @@ struct bus_dma_impl bus_dma_bounce_impl = { .load_kmsan = bounce_bus_dmamap_load_kmsan, #endif }; + +static void * +nonsecure_alloc(uma_zone_t zone, vm_size_t size, int domain, uint8_t *pflag, + int wait) +{ + void *p; /* Returned page */ + + *pflag = UMA_SLAB_KERNEL; + p = kmem_malloc_domainset(DOMAINSET_FIXED(domain), size, + wait | M_UNPROTECTED); + + return (p); +} + +static void +nonsecure_free(void *mem, vm_size_t size, uint8_t flags) +{ + KASSERT((flags & UMA_SLAB_KERNEL) != 0, + ("UMA: page_free used with invalid flags %x", flags)); + + kmem_free(mem, size); +} + +static void +busdma_bounce_init(void *dummy) +{ + if (in_realm()) + nonsecure_allocator = busdma_bufalloc_create("nonsecure", + dcache_line_size, /* minimum_alignment */ + nonsecure_alloc, /* uma_alloc func */ + nonsecure_free, /* uma_free func */ + UMA_ZONE_NOTOUCH); /* uma_zcreate_flags */ + else + nonsecure_allocator = busdma_bufalloc_create("nonsecure", + dcache_line_size, /* minimum_alignment */ + NULL, /* uma_alloc func */ + NULL, /* uma_free func */ + UMA_ZONE_NOTOUCH); /* uma_zcreate_flags */ +} + +SYSINIT(busdma, SI_SUB_KMEM + 1, SI_ORDER_FIRST, busdma_bounce_init, NULL); diff --git a/sys/arm64/arm64/efirt_machdep.c b/sys/arm64/arm64/efirt_machdep.c index b079393cd2a9..07946a2bdb82 100644 --- a/sys/arm64/arm64/efirt_machdep.c +++ b/sys/arm64/arm64/efirt_machdep.c @@ -46,6 +46,7 @@ #include <sys/vmmeter.h> #include <machine/pte.h> +#include <machine/rsi.h> #include <machine/vmparam.h> #include <vm/vm.h> @@ -171,6 +172,7 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz) vm_page_t efi_l0_page; uint64_t idx; int i, mode; + uint64_t pa; obj_1t1_pt = vm_pager_allocate(OBJT_PHYS, NULL, L0_ENTRIES + L0_ENTRIES * Ln_ENTRIES + L0_ENTRIES * Ln_ENTRIES * Ln_ENTRIES + @@ -231,7 +233,13 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz) for (va = p->md_phys, idx = 0; idx < p->md_pages; idx += (PAGE_SIZE / EFI_PAGE_SIZE), va += PAGE_SIZE) { l3 = efi_1t1_l3(va); - *l3 = va | l3_attr; + + if (mode == VM_MEMATTR_DEVICE && in_realm()) + pa = va | prot_ns_shared_pa; + else + pa = va; + + *l3 = PHYS_TO_PTE(pa) | l3_attr; } VM_OBJECT_WUNLOCK(obj_1t1_pt); } diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 2d2982fdfae7..adc583812e5b 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -152,6 +152,7 @@ #include <machine/machdep.h> #include <machine/md_var.h> #include <machine/pcb.h> +#include <machine/rsi.h> #ifdef NUMA #define PMAP_MEMDOM MAXMEMDOM @@ -539,6 +540,9 @@ static int pmap_bti_copy(pmap_t dst_pmap, pmap_t src_pmap); static void pmap_bti_deassign_all(pmap_t pmap); static void pagezero(void *); +static void pmap_set_protected(pt_entry_t old_l3); +static void pmap_set_unprotected(pt_entry_t new_l3); + /* * These load the old table data and store the new value. * They need to be atomic as the System MMU may write to the table at @@ -2381,6 +2385,11 @@ pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode) KASSERT((size & PAGE_MASK) == 0, ("pmap_kenter: Mapping is not page-sized")); + /* CCA - Map devices as nonsecure */ + if (in_realm() && (mode == VM_MEMATTR_DEVICE || + mode == VM_MEMATTR_DEVICE_NP)) + pa |= prot_ns_shared_pa; + attr = ATTR_AF | pmap_sh_attr | ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(mode); old_l3e = 0; @@ -4224,6 +4233,9 @@ pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva, if ((old_l3 & ATTR_SW_WIRED) != 0) pmap->pm_stats.wired_count--; pmap_resident_count_dec(pmap, 1); + /* Below will only be true in a realm environment. */ + if (PTE_TO_PHYS(old_l3) & prot_ns_shared_pa) + pmap_set_protected(old_l3); if ((old_l3 & ATTR_SW_MANAGED) != 0) { m = PTE_TO_VM_PAGE(old_l3); if (pmap_pte_dirty(pmap, old_l3)) @@ -5376,6 +5388,28 @@ restart: return (KERN_SUCCESS); } +static void +pmap_set_unprotected(pt_entry_t new_l3) +{ + vm_paddr_t pa; + + pa = PTE_TO_PHYS(new_l3) & ~prot_ns_shared_pa; + + rsi_set_addr_range_state(pa, pa + L3_SIZE, RSI_RIPAS_EMPTY, + RSI_CHANGE_DESTROYED, NULL); +} + +static void +pmap_set_protected(pt_entry_t old_l3) +{ + vm_paddr_t pa; + + pa = PTE_TO_PHYS(old_l3) & ~prot_ns_shared_pa; + + rsi_set_addr_range_state(pa, pa + L3_SIZE, RSI_RIPAS_RAM, + RSI_CHANGE_DESTROYED, NULL); +} + /* * Insert the given physical page (p) at * the specified virtual address (v) in the @@ -5409,6 +5443,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, if ((m->oflags & VPO_UNMANAGED) == 0) VM_PAGE_OBJECT_BUSY_ASSERT(m); pa = VM_PAGE_TO_PHYS(m); + if (in_realm() && (flags & PMAP_ENTER_UNPROTECTED) != 0) + pa |= prot_ns_shared_pa; new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr | L3_PAGE); new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr); @@ -5728,6 +5764,10 @@ validate: #endif rv = KERN_SUCCESS; + + if (in_realm() && (flags & PMAP_ENTER_UNPROTECTED) != 0) + pmap_set_unprotected(new_l3); + out: if (lock != NULL) rw_wunlock(lock); diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 8f550a644db6..faecea51d5c1 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -9,6 +9,7 @@ kern/subr_devmap.c standard kern/subr_efi_map.c standard kern/subr_intr.c optional intrng kern/subr_physmem.c standard +kern/subr_busdma_bufalloc.c standard libkern/strlen.c standard libkern/arm64/crc32c_armv8.S standard diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index bdea77bfabf0..233292bdd6cb 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -110,6 +110,10 @@ #include <vm/uma_int.h> #include <machine/md_var.h> +#if defined(__aarch64__) +#include <machine/pmap.h> +#include <machine/rsi.h> +#endif struct vm_domain vm_dom[MAXMEMDOM]; @@ -1290,6 +1294,10 @@ PHYS_TO_VM_PAGE(vm_paddr_t pa) vm_page_t m; #ifdef VM_PHYSSEG_SPARSE +#if defined(__aarch64__) + if (in_realm()) + pa &= ~prot_ns_shared_pa; /* Mask off secure bit */ +#endif m = vm_phys_paddr_to_vm_page(pa); if (m == NULL) m = vm_phys_fictitious_to_vm_page(pa);
