There are cases where normal kernel pages tables, i.e. idmap_pg_dir and swapper_pg_dir are not sufficient because they may be overwritten.
This happens when we transition from one world to another: for example during kexec kernel relocation transition, and also during hibernate kernel restore transition. In these cases, if MMU is needed, the page table memory must be allocated from a safe place. Transitional tables is intended to allow just that. Signed-off-by: Pavel Tatashin <[email protected]> --- arch/arm64/Kconfig | 4 + arch/arm64/include/asm/pgtable-hwdef.h | 1 + arch/arm64/include/asm/trans_table.h | 66 ++++++ arch/arm64/mm/Makefile | 1 + arch/arm64/mm/trans_table.c | 272 +++++++++++++++++++++++++ 5 files changed, 344 insertions(+) create mode 100644 arch/arm64/include/asm/trans_table.h create mode 100644 arch/arm64/mm/trans_table.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3adcec05b1f6..91a7416ffe4e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -999,6 +999,10 @@ config CRASH_DUMP For more details see Documentation/admin-guide/kdump/kdump.rst +config TRANS_TABLE + def_bool y + depends on HIBERNATION || KEXEC_CORE + config XEN_DOM0 def_bool y depends on XEN diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index db92950bb1a0..dcb4f13c7888 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -110,6 +110,7 @@ #define PUD_TABLE_BIT (_AT(pudval_t, 1) << 1) #define PUD_TYPE_MASK (_AT(pudval_t, 3) << 0) #define PUD_TYPE_SECT (_AT(pudval_t, 1) << 0) +#define PUD_SECT_RDONLY (_AT(pudval_t, 1) << 7) /* AP[2] */ /* * Level 2 descriptor (PMD). diff --git a/arch/arm64/include/asm/trans_table.h b/arch/arm64/include/asm/trans_table.h new file mode 100644 index 000000000000..4d7bd0bf36c0 --- /dev/null +++ b/arch/arm64/include/asm/trans_table.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019, Microsoft Corporation. + * Pavel Tatashin <[email protected]> + */ + +#ifndef _ASM_TRANS_TABLE_H +#define _ASM_TRANS_TABLE_H + +#include <asm/pgtable-types.h> + +/* + * trans_alloc_page + * - Allocator that should return exactly one uninitilaized page, if this + * allocator fails, trans_table returns -ENOMEM error. + * + * trans_alloc_arg + * - Passed to trans_alloc_page as an argument + * + * trans_flags + * - bitmap with flags that control how page table is filled. + * TRANS_MKWRITE: during page table copy make PTE, PME, and PUD page + * writeable by removing RDONLY flag from PTE. + * TRANS_MKVALID: during page table copy, if PTE present, but not valid, + * make it valid. + * TRANS_CHECKPFN: During page table copy, for every PTE entry check that + * PFN that this PTE points to is valid. Otherwise return + * -ENXIO + * TRANS_FORCEMAP: During page map, if translation exists, force + * overwrite it. Otherwise -ENXIO may be returned by + * trans_table_map_* functions if conflict is detected. + */ + +#define TRANS_MKWRITE (1 << 0) +#define TRANS_MKVALID (1 << 1) +#define TRANS_CHECKPFN (1 << 2) +#define TRANS_FORCEMAP (1 << 3) + +struct trans_table_info { + void * (*trans_alloc_page)(void *); + void *trans_alloc_arg; + unsigned long trans_flags; +}; + +/* Create and empty trans table. */ +int trans_table_create_empty(struct trans_table_info *info, + pgd_t **trans_table); + +/* + * Create trans table and copy entries from from_table to trans_table in range + * [start, end) + */ +int trans_table_create_copy(struct trans_table_info *info, pgd_t **trans_table, + pgd_t *from_table, unsigned long start, + unsigned long end); + +/* + * Add map entry to trans_table for a base-size page at PTE level. + * page: page to be mapped. + * dst_addr: new VA address for the pages + * pgprot: protection for the page. + */ +int trans_table_map_page(struct trans_table_info *info, pgd_t *trans_table, + void *page, unsigned long dst_addr, pgprot_t pgprot); + +#endif /* _ASM_TRANS_TABLE_H */ diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 849c1df3d214..3794fff18659 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -6,6 +6,7 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_ARM64_PTDUMP_CORE) += dump.o obj-$(CONFIG_ARM64_PTDUMP_DEBUGFS) += ptdump_debugfs.o +obj-$(CONFIG_TRANS_TABLE) += trans_table.o obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/trans_table.c b/arch/arm64/mm/trans_table.c new file mode 100644 index 000000000000..d5729eb318b7 --- /dev/null +++ b/arch/arm64/mm/trans_table.c @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019, Microsoft Corporation. + * Pavel Tatashin <[email protected]> + */ + +/* + * Transitional tables are used during system transferring from one world to + * another: such as during hibernate restore, and kexec reboots. During these + * phases one cannot rely on page table not being overwritten. + * + */ + +#include <asm/trans_table.h> +#include <asm/pgalloc.h> +#include <asm/pgtable.h> + +static void *trans_alloc(struct trans_table_info *info) +{ + void *page = info->trans_alloc_page(info->trans_alloc_arg); + + if (page) + clear_page(page); + + return page; +} + +static int trans_table_copy_pte(struct trans_table_info *info, pte_t *dst_ptep, + pte_t *src_ptep, unsigned long start, + unsigned long end) +{ + unsigned long addr = start; + int i = pgd_index(addr); + + do { + pte_t src_pte = READ_ONCE(src_ptep[i]); + + if (pte_none(src_pte)) + continue; + if (info->trans_flags & TRANS_MKWRITE) + src_pte = pte_mkwrite(src_pte); + if (info->trans_flags & TRANS_MKVALID) + src_pte = pte_mkpresent(src_pte); + if (info->trans_flags & TRANS_CHECKPFN) { + if (!pfn_valid(pte_pfn(src_pte))) + return -ENXIO; + } + set_pte(&dst_ptep[i], src_pte); + } while (addr += PAGE_SIZE, i++, addr != end && i < PTRS_PER_PTE); + + return 0; +} + +static int trans_table_copy_pmd(struct trans_table_info *info, pmd_t *dst_pmdp, + pmd_t *src_pmdp, unsigned long start, + unsigned long end) +{ + unsigned long next; + unsigned long addr = start; + int i = pgd_index(addr); + int rc; + + do { + pmd_t src_pmd = READ_ONCE(src_pmdp[i]); + pmd_t dst_pmd = READ_ONCE(dst_pmdp[i]); + pte_t *dst_ptep, *src_ptep; + + next = pmd_addr_end(addr, end); + if (pmd_none(src_pmd)) + continue; + + if (!pmd_table(src_pmd)) { + if (info->trans_flags & TRANS_MKWRITE) + pmd_val(src_pmd) &= ~PMD_SECT_RDONLY; + set_pmd(&dst_pmdp[i], src_pmd); + continue; + } + + if (pmd_none(dst_pmd)) { + pte_t *t = trans_alloc(info); + + if (!t) + return -ENOMEM; + + __pmd_populate(&dst_pmdp[i], __pa(t), PTE_TYPE_PAGE); + dst_pmd = READ_ONCE(dst_pmdp[i]); + } + + src_ptep = __va(pmd_page_paddr(src_pmd)); + dst_ptep = __va(pmd_page_paddr(dst_pmd)); + + rc = trans_table_copy_pte(info, dst_ptep, src_ptep, addr, next); + if (rc) + return rc; + } while (addr = next, i++, addr != end && i < PTRS_PER_PMD); + + return 0; +} + +static int trans_table_copy_pud(struct trans_table_info *info, pud_t *dst_pudp, + pud_t *src_pudp, unsigned long start, + unsigned long end) +{ + unsigned long next; + unsigned long addr = start; + int i = pgd_index(addr); + int rc; + + do { + pud_t src_pud = READ_ONCE(src_pudp[i]); + pud_t dst_pud = READ_ONCE(dst_pudp[i]); + pmd_t *dst_pmdp, *src_pmdp; + + next = pud_addr_end(addr, end); + if (pud_none(src_pud)) + continue; + + if (!pud_table(src_pud)) { + if (info->trans_flags & TRANS_MKWRITE) + pud_val(src_pud) &= ~PUD_SECT_RDONLY; + set_pud(&dst_pudp[i], src_pud); + continue; + } + + if (pud_none(dst_pud)) { + pmd_t *t = trans_alloc(info); + + if (!t) + return -ENOMEM; + + __pud_populate(&dst_pudp[i], __pa(t), PMD_TYPE_TABLE); + dst_pud = READ_ONCE(dst_pudp[i]); + } + + src_pmdp = __va(pud_page_paddr(src_pud)); + dst_pmdp = __va(pud_page_paddr(dst_pud)); + + rc = trans_table_copy_pmd(info, dst_pmdp, src_pmdp, addr, next); + if (rc) + return rc; + } while (addr = next, i++, addr != end && i < PTRS_PER_PUD); + + return 0; +} + +static int trans_table_copy_pgd(struct trans_table_info *info, pgd_t *dst_pgdp, + pgd_t *src_pgdp, unsigned long start, + unsigned long end) +{ + unsigned long next; + unsigned long addr = start; + int i = pgd_index(addr); + int rc; + + do { + pgd_t src_pgd; + pgd_t dst_pgd; + pud_t *dst_pudp, *src_pudp; + + src_pgd = READ_ONCE(src_pgdp[i]); + dst_pgd = READ_ONCE(dst_pgdp[i]); + next = pgd_addr_end(addr, end); + if (pgd_none(src_pgd)) + continue; + + if (pgd_none(dst_pgd)) { + pud_t *t = trans_alloc(info); + + if (!t) + return -ENOMEM; + + __pgd_populate(&dst_pgdp[i], __pa(t), PUD_TYPE_TABLE); + dst_pgd = READ_ONCE(dst_pgdp[i]); + } + + src_pudp = __va(pgd_page_paddr(src_pgd)); + dst_pudp = __va(pgd_page_paddr(dst_pgd)); + + rc = trans_table_copy_pud(info, dst_pudp, src_pudp, addr, next); + if (rc) + return rc; + } while (addr = next, i++, addr != end && i < PTRS_PER_PGD); + + return 0; +} + +int trans_table_create_empty(struct trans_table_info *info, pgd_t **trans_table) +{ + pgd_t *dst_pgdp = trans_alloc(info); + + if (!dst_pgdp) + return -ENOMEM; + + *trans_table = dst_pgdp; + + return 0; +} + +int trans_table_create_copy(struct trans_table_info *info, pgd_t **trans_table, + pgd_t *from_table, unsigned long start, + unsigned long end) +{ + int rc; + + rc = trans_table_create_empty(info, trans_table); + if (rc) + return rc; + + return trans_table_copy_pgd(info, *trans_table, from_table, start, end); +} + +int trans_table_map_page(struct trans_table_info *info, pgd_t *trans_table, + void *page, unsigned long dst_addr, pgprot_t pgprot) +{ + int pgd_idx = pgd_index(dst_addr); + int pud_idx = pud_index(dst_addr); + int pmd_idx = pmd_index(dst_addr); + int pte_idx = pte_index(dst_addr); + pgd_t *pgdp = trans_table; + pgd_t pgd = READ_ONCE(pgdp[pgd_idx]); + pud_t *pudp, pud; + pmd_t *pmdp, pmd; + pte_t *ptep, pte; + + if (pgd_none(pgd)) { + pud_t *t = trans_alloc(info); + + if (!t) + return -ENOMEM; + + __pgd_populate(&pgdp[pgd_idx], __pa(t), PUD_TYPE_TABLE); + pgd = READ_ONCE(pgdp[pgd_idx]); + } + + pudp = __va(pgd_page_paddr(pgd)); + pud = READ_ONCE(pudp[pud_idx]); + if (pud_sect(pud) && !(info->trans_flags & TRANS_FORCEMAP)) { + return -ENXIO; + } else if (pud_none(pud) || pud_sect(pud)) { + pmd_t *t = trans_alloc(info); + + if (!t) + return -ENOMEM; + + __pud_populate(&pudp[pud_idx], __pa(t), PMD_TYPE_TABLE); + pud = READ_ONCE(pudp[pud_idx]); + } + + pmdp = __va(pud_page_paddr(pud)); + pmd = READ_ONCE(pmdp[pmd_idx]); + if (pmd_sect(pmd) && !(info->trans_flags & TRANS_FORCEMAP)) { + return -ENXIO; + } else if (pmd_none(pmd) || pmd_sect(pmd)) { + pte_t *t = trans_alloc(info); + + if (!t) + return -ENOMEM; + + __pmd_populate(&pmdp[pmd_idx], __pa(t), PTE_TYPE_PAGE); + pmd = READ_ONCE(pmdp[pmd_idx]); + } + + ptep = __va(pmd_page_paddr(pmd)); + pte = READ_ONCE(ptep[pte_idx]); + + if (!pte_none(pte) && !(info->trans_flags & TRANS_FORCEMAP)) + return -ENXIO; + + set_pte(&ptep[pte_idx], pfn_pte(virt_to_pfn(page), pgprot)); + + return 0; +} -- 2.22.0
