This is an automated email from the ASF dual-hosted git repository. acassis pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nuttx.git
commit 6e15994f4cb0e19eeaf7d9c470b8613d94ee1f4b Author: Ville Juven <[email protected]> AuthorDate: Thu Sep 12 11:17:19 2024 +0300 arm64_addrenv: Add support for 4 level MMU translations The original code made the incorrect assumption that the amount of translation levels is 3, but this is incorrect. The amount of levels is 4 and the amount of levels that are utilized / in use is set dynamically from the amount of VA bits in use. --- arch/arm64/src/common/arm64_addrenv.c | 72 ++++++++++++++++++----------- arch/arm64/src/common/arm64_addrenv_perms.c | 2 +- arch/arm64/src/common/arm64_addrenv_pgmap.c | 12 +++-- arch/arm64/src/common/arm64_addrenv_utils.c | 11 +++-- arch/arm64/src/common/arm64_mmu.c | 5 ++ arch/arm64/src/common/arm64_mmu.h | 22 +++++++++ 6 files changed, 88 insertions(+), 36 deletions(-) diff --git a/arch/arm64/src/common/arm64_addrenv.c b/arch/arm64/src/common/arm64_addrenv.c index 5f28883774..e414dc1626 100644 --- a/arch/arm64/src/common/arm64_addrenv.c +++ b/arch/arm64/src/common/arm64_addrenv.c @@ -115,24 +115,28 @@ static void map_spgtables(arch_addrenv_t *addrenv, uintptr_t vaddr) { int i; uintptr_t prev; + uintptr_t l0; - /* Start from L1, and connect until max level - 1 */ + /* Get the base page table level and the page table associated with it */ - prev = arm64_pgvaddr(addrenv->spgtables[0]); + l0 = mmu_get_base_pgt_level(); + prev = arm64_pgvaddr(addrenv->spgtables[l0]); - /* Check if the mapping already exists */ + /* Start from the base level, and connect until max level - 1 */ - if (mmu_ln_getentry(1, prev, vaddr) != 0) + for (i = l0; i < (ARCH_SPGTS - 1); i++) { - return; - } + uintptr_t next = addrenv->spgtables[i + 1]; - /* No mapping yet, create it */ + /* Check if the mapping already exists */ + + if (mmu_ln_getentry(i, prev, vaddr) == 0) + { + /* No mapping yet, create it */ + + mmu_ln_setentry(i, prev, next, vaddr, MMU_UPGT_FLAGS); + } - for (i = 0; i < (ARCH_SPGTS - 1); i++) - { - uintptr_t next = addrenv->spgtables[i + 1]; - mmu_ln_setentry(i + 1, prev, next, vaddr, MMU_UPGT_FLAGS); prev = arm64_pgvaddr(next); } } @@ -157,7 +161,7 @@ static int create_spgtables(arch_addrenv_t *addrenv) int i; uintptr_t paddr; - for (i = 0; i < ARCH_SPGTS; i++) + for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++) { paddr = mm_pgalloc(1); if (!paddr) @@ -197,20 +201,37 @@ static int create_spgtables(arch_addrenv_t *addrenv) static int copy_kernel_mappings(arch_addrenv_t *addrenv) { - uintptr_t user_mappings = arm64_pgvaddr(addrenv->spgtables[0]); + uintptr_t kpgt; + uintptr_t upgt; + uintptr_t l0; - /* Copy the L1 references */ + /* Determine the base page table level */ - if (user_mappings == 0) + l0 = mmu_get_base_pgt_level(); + kpgt = g_kernel_mappings; + + /* Don't copy L0 references, as those encompass 512GB each */ + + if (l0 == 0) + { + upgt = arm64_pgvaddr(addrenv->spgtables[1]); + kpgt = arm64_pgvaddr(mmu_pte_to_paddr(((uintptr_t *)kpgt)[0])); + } + else + { + upgt = arm64_pgvaddr(addrenv->spgtables[l0]); + } + + if (upgt == 0 || kpgt == 0) { return -EINVAL; } - memcpy((void *)user_mappings, (void *)g_kernel_mappings, MMU_PAGE_SIZE); + memcpy((void *)upgt, (void *)kpgt, MMU_PAGE_SIZE); /* Update with memory by flushing the cache */ - up_flush_dcache(user_mappings, user_mappings + MMU_PAGE_SIZE); + up_flush_dcache(upgt, upgt + MMU_PAGE_SIZE); return OK; } @@ -248,8 +269,8 @@ static int create_region(arch_addrenv_t *addrenv, uintptr_t vaddr, nmapped = 0; npages = MM_NPAGES(size); - ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); - ptlevel = ARCH_SPGTS; + ptlevel = MMU_PGT_LEVEL_MAX - 1; + ptprev = arm64_pgvaddr(addrenv->spgtables[ptlevel]); /* Create mappings for the lower level tables */ @@ -384,6 +405,7 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize, uintptr_t textbase; uintptr_t database; uintptr_t heapbase; + uintptr_t l0; DEBUGASSERT(addrenv); DEBUGASSERT(MM_ISALIGNED(ARCH_ADDRENV_VBASE)); @@ -485,7 +507,8 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize, /* Provide the ttbr0 value for context switch */ - addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[0], 0); + l0 = mmu_get_base_pgt_level(); + addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0); /* Synchronize data and instruction pipelines */ @@ -532,17 +555,14 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv) /* Things start from the beginning of the user virtual memory */ vaddr = ARCH_ADDRENV_VBASE; - pgsize = mmu_get_region_size(ARCH_SPGTS); + pgsize = mmu_get_region_size(MMU_PGT_LEVEL_MAX - 1); /* First destroy the allocated memory and the final level page table */ ptprev = (uintptr_t *)arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); if (ptprev) { - /* walk user space only */ - - i = (ARCH_SPGTS < 2) ? vaddr / pgsize : 0; - for (; i < ENTRIES_PER_PGT; i++, vaddr += pgsize) + for (i = 0; i < ENTRIES_PER_PGT; i++, vaddr += pgsize) { ptlast = (uintptr_t *)arm64_pgvaddr(mmu_pte_to_paddr(ptprev[i])); if (ptlast) @@ -570,7 +590,7 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv) /* Then destroy the static tables */ - for (i = 0; i < ARCH_SPGTS; i++) + for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++) { paddr = addrenv->spgtables[i]; if (paddr) diff --git a/arch/arm64/src/common/arm64_addrenv_perms.c b/arch/arm64/src/common/arm64_addrenv_perms.c index f68e98f49e..441b0154d1 100644 --- a/arch/arm64/src/common/arm64_addrenv_perms.c +++ b/arch/arm64/src/common/arm64_addrenv_perms.c @@ -70,7 +70,7 @@ static int modify_region(uintptr_t vstart, uintptr_t vend, uintptr_t setmask) for (vaddr = vstart; vaddr < vend; vaddr += MM_PGSIZE) { - for (ptlevel = 1, lnvaddr = l1vaddr; + for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = l1vaddr; ptlevel < MMU_PGT_LEVEL_MAX; ptlevel++) { diff --git a/arch/arm64/src/common/arm64_addrenv_pgmap.c b/arch/arm64/src/common/arm64_addrenv_pgmap.c index f7b810c849..e53528b97f 100644 --- a/arch/arm64/src/common/arm64_addrenv_pgmap.c +++ b/arch/arm64/src/common/arm64_addrenv_pgmap.c @@ -90,7 +90,9 @@ uintptr_t up_addrenv_find_page(arch_addrenv_t *addrenv, uintptr_t vaddr) /* Make table walk to find the page */ - for (ptlevel = 1, lnvaddr = pgdir; ptlevel < MMU_PGT_LEVEL_MAX; ptlevel++) + for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = pgdir; + ptlevel < MMU_PGT_LEVEL_MAX; + ptlevel++) { paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr)); lnvaddr = arm64_pgvaddr(paddr); @@ -189,6 +191,7 @@ int up_addrenv_kmap_init(void) struct arch_addrenv_s *addrenv; uintptr_t next; uintptr_t vaddr; + uintptr_t l0; int i; /* Populate the static page tables one by one */ @@ -196,19 +199,20 @@ int up_addrenv_kmap_init(void) addrenv = &g_kernel_addrenv; next = g_kernel_pgt_pbase; vaddr = CONFIG_ARCH_KMAP_VBASE; + l0 = mmu_get_base_pgt_level(); - for (i = 0; i < ARCH_SPGTS; i++) + for (i = l0; i < ARCH_SPGTS; i++) { /* Connect the static page tables */ uintptr_t lnvaddr = arm64_pgvaddr(next); addrenv->spgtables[i] = next; - next = mmu_pte_to_paddr(mmu_ln_getentry(i + 1, lnvaddr, vaddr)); + next = mmu_pte_to_paddr(mmu_ln_getentry(i, lnvaddr, vaddr)); } /* Set the page directory root */ - addrenv->ttbr0 = mmu_ttbr_reg(g_kernel_pgt_pbase, 0); + addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0); /* When all is set and done, flush the data caches */ diff --git a/arch/arm64/src/common/arm64_addrenv_utils.c b/arch/arm64/src/common/arm64_addrenv_utils.c index bdfaeaa253..2a95765148 100644 --- a/arch/arm64/src/common/arm64_addrenv_utils.c +++ b/arch/arm64/src/common/arm64_addrenv_utils.c @@ -66,8 +66,8 @@ uintptr_t arm64_get_pgtable(arch_addrenv_t *addrenv, uintptr_t vaddr) /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */ - ptlevel = ARCH_SPGTS; - ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + ptlevel = MMU_PGT_LEVEL_MAX - 1; + ptprev = arm64_pgvaddr(addrenv->spgtables[ptlevel]); if (!ptprev) { /* Something is very wrong */ @@ -184,7 +184,10 @@ int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr, uintptr_t ptlevel; uintptr_t paddr; - ptprev = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]); + /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */ + + ptlevel = MMU_PGT_LEVEL_MAX - 1; + ptprev = arm64_pgvaddr(addrenv->spgtables[ptlevel]); if (!ptprev) { /* Something is very wrong */ @@ -192,8 +195,6 @@ int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr, return -EFAULT; } - ptlevel = ARCH_SPGTS; - /* Remove the references from the caller's address environment */ for (; npages > 0; npages--) diff --git a/arch/arm64/src/common/arm64_mmu.c b/arch/arm64/src/common/arm64_mmu.c index 6588ad4534..953f139cf4 100644 --- a/arch/arm64/src/common/arm64_mmu.c +++ b/arch/arm64/src/common/arm64_mmu.c @@ -787,3 +787,8 @@ size_t mmu_get_region_size(uint32_t ptlevel) return g_pgt_sizes[ptlevel]; } + +uintptr_t mmu_get_base_pgt_level(void) +{ + return XLAT_TABLE_BASE_LEVEL; +} diff --git a/arch/arm64/src/common/arm64_mmu.h b/arch/arm64/src/common/arm64_mmu.h index b75d2e8ef3..1fae9d0338 100644 --- a/arch/arm64/src/common/arm64_mmu.h +++ b/arch/arm64/src/common/arm64_mmu.h @@ -628,6 +628,28 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr, size_t mmu_get_region_size(uint32_t ptlevel); +/**************************************************************************** + * Name: mmu_get_base_pgt_level + * + * Description: + * Get the base translation table level. The ARM64 MMU implementation + * optimizes the amount of translation table levels in use, based on the + * configured virtual address range (CONFIG_ARM64_VA_BITS). + * + * Table indices range from 0...3 and the lowest table indices are dropped + * as needed. If CONFIG_ARM64_VA_BITS >= 40, all 4 translation table levels + * are needed. + * + * Input Parameters: + * None. + * + * Returned Value: + * The base translation table level. + * + ****************************************************************************/ + +uintptr_t mmu_get_base_pgt_level(void); + #endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM64_SRC_COMMON_ARM64_MMU_H */
