Writing the PTE with a normal non-volatile access is a bad idea, because speculation may already use a partially written entry.
Use WRITE_ONCE to turn this into a volatile write via a new set_pte helper. The extra helper is useful for type enforcement and for future extension when we start to consistently apply break-before-make[1]. [1]: https://lore.kernel.org/all/87a5btrcyr.wl-...@kernel.org/ Signed-off-by: Ahmad Fatoum <a.fat...@pengutronix.de> --- arch/arm/cpu/mmu_32.c | 40 +++++++++++++++++++++++++++++----------- arch/arm/cpu/mmu_64.c | 20 +++++++++++++------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/arch/arm/cpu/mmu_32.c b/arch/arm/cpu/mmu_32.c index ec6bd27da4e1..9f50194c7c2b 100644 --- a/arch/arm/cpu/mmu_32.c +++ b/arch/arm/cpu/mmu_32.c @@ -65,6 +65,11 @@ static bool pgd_type_table(u32 pgd) #define PTE_SIZE (PTRS_PER_PTE * sizeof(u32)) +static void set_pte(uint32_t *pt, uint32_t val) +{ + WRITE_ONCE(*pt, val); +} + #ifdef __PBL__ static uint32_t *alloc_pte(void) { @@ -141,13 +146,14 @@ static u32 *arm_create_pte(unsigned long virt, unsigned long phys, ttb_idx = pgd_index(virt); for (i = 0; i < PTRS_PER_PTE; i++) { - table[i] = phys | PTE_TYPE_SMALL | flags; + set_pte(&table[i], phys | PTE_TYPE_SMALL | flags); virt += PAGE_SIZE; phys += PAGE_SIZE; } dma_flush_range(table, PTRS_PER_PTE * sizeof(u32)); - ttb[ttb_idx] = (unsigned long)table | PMD_TYPE_TABLE; + // TODO break-before-make missing + set_pte(&ttb[ttb_idx], (unsigned long)table | PMD_TYPE_TABLE); dma_flush_range(&ttb[ttb_idx], sizeof(u32)); return table; @@ -264,14 +270,17 @@ static void __arch_remap_range(void *_virt_addr, phys_addr_t phys_addr, size_t s if (size >= PGDIR_SIZE && pgdir_size_aligned && IS_ALIGNED(phys_addr, PGDIR_SIZE) && !pgd_type_table(*pgd)) { + u32 val; /* * TODO: Add code to discard a page table and * replace it with a section */ chunk = PGDIR_SIZE; - *pgd = phys_addr | pmd_flags; + val = phys_addr | pmd_flags; if (map_type != MAP_FAULT) - *pgd |= PMD_TYPE_SECT; + val |= PMD_TYPE_SECT; + // TODO break-before-make missing + set_pte(pgd, val); dma_flush_range(pgd, sizeof(*pgd)); } else { unsigned int num_ptes; @@ -310,10 +319,15 @@ static void __arch_remap_range(void *_virt_addr, phys_addr_t phys_addr, size_t s } for (i = 0; i < num_ptes; i++) { - pte[i] = phys_addr + i * PAGE_SIZE; - pte[i] |= pte_flags; + u32 val; + + val = phys_addr + i * PAGE_SIZE; + val |= pte_flags; if (map_type != MAP_FAULT) - pte[i] |= PTE_TYPE_SMALL; + val |= PTE_TYPE_SMALL; + + // TODO break-before-make missing + set_pte(&pte[i], val); } dma_flush_range(pte, num_ptes * sizeof(u32)); @@ -350,7 +364,7 @@ static void create_sections(unsigned long first, unsigned long last, unsigned int i, addr = first; for (i = ttb_start; i < ttb_end; i++) { - ttb[i] = addr | flags; + set_pte(&ttb[i], addr | flags); addr += PGDIR_SIZE; } } @@ -366,8 +380,10 @@ void *map_io_sections(unsigned long phys, void *_start, size_t size) unsigned long start = (unsigned long)_start, sec; uint32_t *ttb = get_ttb(); - for (sec = start; sec < start + size; sec += PGDIR_SIZE, phys += PGDIR_SIZE) - ttb[pgd_index(sec)] = phys | get_pmd_flags(MAP_UNCACHED); + for (sec = start; sec < start + size; sec += PGDIR_SIZE, phys += PGDIR_SIZE) { + // TODO break-before-make missing + set_pte(&ttb[pgd_index(sec)], phys | get_pmd_flags(MAP_UNCACHED)); + } dma_flush_range(ttb, 0x4000); tlb_invalidate(); @@ -410,7 +426,9 @@ static void create_vector_table(unsigned long adr) vectors, adr); arm_create_pte(adr, adr, get_pte_flags(MAP_UNCACHED)); pte = find_pte(adr); - *pte = (u32)vectors | PTE_TYPE_SMALL | get_pte_flags(MAP_CACHED); + // TODO break-before-make missing + set_pte(pte, (u32)vectors | PTE_TYPE_SMALL | + get_pte_flags(MAP_CACHED)); } arm_fixup_vectors(); diff --git a/arch/arm/cpu/mmu_64.c b/arch/arm/cpu/mmu_64.c index d9b0b74d7314..efd788c93e7b 100644 --- a/arch/arm/cpu/mmu_64.c +++ b/arch/arm/cpu/mmu_64.c @@ -31,12 +31,14 @@ static uint64_t *get_ttb(void) return (uint64_t *)get_ttbr(current_el()); } +static void set_pte(uint64_t *pt, uint64_t val) +{ + WRITE_ONCE(*pt, val); +} + static void set_table(uint64_t *pt, uint64_t *table_addr) { - uint64_t val; - - val = PTE_TYPE_TABLE | (uint64_t)table_addr; - *pt = val; + set_pte(pt, PTE_TYPE_TABLE | (uint64_t)table_addr); } #ifdef __PBL__ @@ -114,14 +116,16 @@ static void split_block(uint64_t *pte, int level) for (i = 0; i < MAX_PTE_ENTRIES; i++) { - new_table[i] = old_pte | (i << levelshift); + set_pte(&new_table[i], old_pte | (i << levelshift)); /* Level 3 block PTEs have the table type */ if ((level + 1) == 3) new_table[i] |= PTE_TYPE_TABLE; } - /* Set the new table into effect */ + /* Set the new table into effect + * TODO: break-before-make missing + */ set_table(pte, new_table); } @@ -157,7 +161,9 @@ static void create_sections(uint64_t virt, uint64_t phys, uint64_t size, IS_ALIGNED(phys, block_size)) { type = (level == 3) ? PTE_TYPE_PAGE : PTE_TYPE_BLOCK; - *pte = phys | attr | type; + + /* TODO: break-before-make missing */ + set_pte(pte, phys | attr | type); addr += block_size; phys += block_size; size -= block_size; -- 2.39.5
