The branch main has been updated by andrew: URL: https://cgit.FreeBSD.org/src/commit/?id=7bb6b62394d3036567cf1453395d9e8b63d3dda1
commit 7bb6b62394d3036567cf1453395d9e8b63d3dda1 Author: Harry Moulton <[email protected]> AuthorDate: 2026-05-18 09:29:03 +0000 Commit: Andrew Turner <[email protected]> CommitDate: 2026-05-27 15:22:26 +0000 arm64: mte: copy/save tags on copy-on-write On a copy-on-write, copy the memory tags from the source pages to the destination pages so the forked process can continue to use MTE. Reviewed by: andrew Sponsored by: Arm Ltd Signed-off-by: Harry Moulton <[email protected]> Differential Revision: https://reviews.freebsd.org/D55955 --- sys/arm64/arm64/mte.c | 41 +++++++++++++++++++++++++++++++++++++++++ sys/arm64/arm64/pmap.c | 13 +++++++++++++ sys/arm64/include/cpu.h | 1 + 3 files changed, 55 insertions(+) diff --git a/sys/arm64/arm64/mte.c b/sys/arm64/arm64/mte.c index ba5d9df3b01c..6e902858a8b9 100644 --- a/sys/arm64/arm64/mte.c +++ b/sys/arm64/arm64/mte.c @@ -49,6 +49,22 @@ static u_int __read_mostly mte_version = 0; struct thread *mte_switch(struct thread *); +#define load_tags(addr) ({ \ + uint64_t __val; \ + asm volatile( \ + ".arch_extension memtag \n" \ + "ldgm %0, [%1] \n" \ + ".arch_extension nomemtag" : "=r" (__val) : "r" (addr)); \ + __val; \ +}) + +#define set_tags(tags, addr) do { \ + asm volatile( \ + ".arch_extension memtag \n" \ + "stgm %0, [%1] \n" \ + ".arch_extension nomemtag" : "=r" (tags) : "r" (addr)); \ +} while (0) + /* Fetch the block size used by tag load and store instructions */ static inline size_t mte_block_size(void) @@ -95,6 +111,31 @@ mte_sync_tags(vm_page_t page) page->md.pv_flags |= PV_MTE_TAGGED; } +/** + * Copy the allocation tags from given target to destination page. This is called + * on a copy-on-write and anything that causes a pmap_copy_page call. + */ +void +mte_copy_tags(vm_page_t srcpage, vm_page_t dstpage, char *src, char *dst) +{ + size_t block_size; + uint64_t tags; + + MPASS((srcpage->md.pv_flags & PV_MTE_TAGGED) != 0); + + /* + * Copy the tags from the source page to the destination page, + * incrementing by the block count read from GMID_EL1 + */ + block_size = mte_block_size(); + for (size_t count = 0; count < PAGE_SIZE; + count += block_size, src += block_size, dst += block_size) { + tags = load_tags(src); + set_tags(tags, dst); + } + dstpage->md.pv_flags |= PV_MTE_TAGGED; +} + void mte_fork(struct thread *new_td, struct thread *orig_td) { diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 6914fc13b065..1fb9ac2011aa 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -146,6 +146,7 @@ #include <vm/uma.h> #include <machine/asan.h> +#include <machine/cpu.h> #include <machine/cpu_feat.h> #include <machine/elf.h> #include <machine/ifunc.h> @@ -6954,6 +6955,15 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst) void *src = VM_PAGE_TO_DMAP(msrc); void *dst = VM_PAGE_TO_DMAP(mdst); + /* + * On a page copy, check whether the src page is tagged. If it is, + * we must copy the tags before copying the contents of the page. + */ + if ((msrc->md.pv_flags & PV_MTE_TAGGED) != 0) + mte_copy_tags(msrc, mdst, src, dst); + else + mdst->md.pv_flags &= ~PV_MTE_TAGGED; + pagecopy(src, dst); } @@ -6970,6 +6980,9 @@ pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], int cnt; while (xfersize > 0) { + KASSERT(ADDR_IS_CANONICAL(a_offset), + ("%s: Address not in canonical form: %lx", __func__, a_offset)); + a_pg_offset = a_offset & PAGE_MASK; m_a = ma[a_offset >> PAGE_SHIFT]; p_a = m_a->phys_addr; diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h index d36e1e56c91c..bdbc601edd26 100644 --- a/sys/arm64/include/cpu.h +++ b/sys/arm64/include/cpu.h @@ -285,6 +285,7 @@ void mte_thread_alloc(struct thread *); void mte_thread0(struct thread *); void mte_sync_tags(vm_page_t page); +void mte_copy_tags(vm_page_t, vm_page_t, char *, char *); /* Functions to read the sanitised view of the special registers */ void update_special_regs(u_int);
