Extend kernel_ident_mapping_init() with ident_pte_init() and a force_pte flag in struct x86_mapping_info so callers can request 4K PTE leaf mappings instead of 2M PMD or 1G PUD leaves.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/x86/include/asm/init.h | 3 +- arch/x86/mm/ident_map.c | 72 ++++++++++++++++++++++++++++++++----- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/init.h b/arch/x86/include/asm/init.h index 01ccdd168df0..d9caed494e7d 100644 --- a/arch/x86/include/asm/init.h +++ b/arch/x86/include/asm/init.h @@ -6,10 +6,11 @@ struct x86_mapping_info { void *(*alloc_pgt_page)(void *); /* allocate buf for page table */ void (*free_pgt_page)(void *, void *); /* free buf for page table */ void *context; /* context for alloc_pgt_page */ - unsigned long page_flag; /* page flag for PMD or PUD entry */ + unsigned long page_flag; /* page flag for PTE, PMD or PUD entry */ unsigned long offset; /* ident mapping offset */ bool direct_gbpages; /* PUD level 1GB page support */ unsigned long kernpg_flag; /* kernel pagetable flag override */ + bool force_pte; /* force 4K PTE mappings */ }; int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c index 5a15bffe6574..83013513ba94 100644 --- a/arch/x86/mm/ident_map.c +++ b/arch/x86/mm/ident_map.c @@ -77,24 +77,73 @@ void kernel_ident_mapping_free(struct x86_mapping_info *info, pgd_t *pgd) info->free_pgt_page(pgd, info->context); } -static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, - unsigned long addr, unsigned long end) +static int ident_pte_init(struct x86_mapping_info *info, pte_t *pte_page, + unsigned long addr, unsigned long end) { - addr &= PMD_MASK; - for (; addr < end; addr += PMD_SIZE) { + addr &= PAGE_MASK; + for (; addr < end; addr += PAGE_SIZE) { + pte_t *pte = pte_page + pte_index(addr); + + if (pte_present(*pte)) + continue; + + set_pte(pte, __pte(((addr - info->offset) | info->page_flag) & ~_PAGE_PSE)); + } + + return 0; +} + +static int ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page, + unsigned long addr, unsigned long end) +{ + unsigned long next; + int result; + + for (; addr < end; addr = next) { pmd_t *pmd = pmd_page + pmd_index(addr); + pte_t *pte; - if (pmd_present(*pmd)) + next = pmd_addr_end(addr, end); + + if (!info->force_pte) { + if (pmd_present(*pmd)) + continue; + + set_pmd(pmd, __pmd(((addr & PMD_MASK) - info->offset) | info->page_flag)); continue; + } - set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag)); + /* if this is already a 2MB page, this portion is already mapped */ + if (pmd_leaf(*pmd)) + continue; + + if (pmd_present(*pmd)) { + pte = pte_offset_kernel(pmd, 0); + result = ident_pte_init(info, pte, addr, next); + if (result) + return result; + continue; + } + + pte = (pte_t *)info->alloc_pgt_page(info->context); + if (!pte) + return -ENOMEM; + + result = ident_pte_init(info, pte, addr, next); + if (result) + return result; + + set_pmd(pmd, __pmd(__pa(pte) | info->kernpg_flag)); } + + return 0; } static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, unsigned long addr, unsigned long end) { unsigned long next; + int result; for (; addr < end; addr = next) { pud_t *pud = pud_page + pud_index(addr); @@ -108,7 +157,7 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, continue; /* Is using a gbpage allowed? */ - use_gbpage = info->direct_gbpages; + use_gbpage = info->direct_gbpages && !info->force_pte; /* Don't use gbpage if it maps more than the requested region. */ /* at the beginning: */ @@ -129,13 +178,17 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page, if (pud_present(*pud)) { pmd = pmd_offset(pud, 0); - ident_pmd_init(info, pmd, addr, next); + result = ident_pmd_init(info, pmd, addr, next); + if (result) + return result; continue; } pmd = (pmd_t *)info->alloc_pgt_page(info->context); if (!pmd) return -ENOMEM; - ident_pmd_init(info, pmd, addr, next); + result = ident_pmd_init(info, pmd, addr, next); + if (result) + return result; set_pud(pud, __pud(__pa(pmd) | info->kernpg_flag)); } @@ -189,6 +242,7 @@ int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page, /* Filter out unsupported __PAGE_KERNEL_* bits: */ info->kernpg_flag &= __default_kernel_pte_mask; + info->page_flag &= __default_kernel_pte_mask; for (; addr < end; addr = next) { pgd_t *pgd = pgd_page + pgd_index(addr); -- 2.55.0.1082.g2b9226bbc0-goog
