Introduce mark_memory_unaccepted() to mark a physical memory range as unaccepted in the EFI unaccepted memory bitmap. Use unaccepted_memory_order() to query the page order matching the EFI unaccepted memory unit_size granularity.
Synchronize mark_memory_unaccepted() with concurrent accept_memory() callers via accepting_list and only invoke arch_unaccept_memory() on sub-ranges that are currently marked accepted in the bitmap. On platforms with dynamic unacceptance (arch_has_unaccept_memory()), load_unaligned_zeropad() across page boundaries is recovered via EX_TYPE_ZEROPAD exception fixup. Thus, skip extending accept_memory() and range_contains_unaccepted_memory() by an extra unit_size guard block to avoid corrupting the acceptance state of adjacent 2MB blocks. Signed-off-by: Kevin Loughlin <[email protected]> --- drivers/firmware/efi/unaccepted_memory.c | 107 ++++++++++++++++++++++- include/linux/mm.h | 12 +++ 2 files changed, 115 insertions(+), 4 deletions(-) diff --git a/drivers/firmware/efi/unaccepted_memory.c b/drivers/firmware/efi/unaccepted_memory.c index 4a8ec8d6a571..128bc1b113cd 100644 --- a/drivers/firmware/efi/unaccepted_memory.c +++ b/drivers/firmware/efi/unaccepted_memory.c @@ -5,6 +5,8 @@ #include <linux/spinlock.h> #include <linux/crash_dump.h> #include <linux/nmi.h> +#include <linux/kernel.h> +#include <linux/mm.h> #include <asm/unaccepted_memory.h> /* Protects unaccepted memory bitmap and accepting_list */ @@ -86,7 +88,16 @@ void accept_memory(phys_addr_t start, unsigned long size) * if 'size+end' is aligned on a unit_size boundary. (immediately * following this comment) */ - if (!(end % unit_size)) + /* + * On platforms without implemented dynamic unacceptance support, + * load_unaligned_zeropad() across a page boundary into unaccepted + * memory causes a fatal VMM exit rather than a recoverable exception. + * When dynamic unacceptance is supported (arch_has_unaccept_memory()), + * load_unaligned_zeropad() is recovered via EX_TYPE_ZEROPAD exception + * fixup, whereas extending the range by unit_size would accept + * adjacent dynamically-unaccepted blocks. + */ + if (!arch_has_unaccept_memory() && !(end % unit_size)) end += unit_size; /* Make sure not to overrun the bitmap */ @@ -191,10 +202,11 @@ bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size) end -= unaccepted->phys_base; /* - * Also consider the unaccepted state of the *next* page. See fix #1 in - * the comment on load_unaligned_zeropad() in accept_memory(). + * Also consider the unaccepted state of the *next* page on platforms + * without implemented dynamic unacceptance support. See fix #1 in the + * comment on load_unaligned_zeropad() in accept_memory(). */ - if (!(end % unit_size)) + if (!arch_has_unaccept_memory() && !(end % unit_size)) end += unit_size; /* Make sure not to overrun the bitmap */ @@ -215,6 +227,93 @@ bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size) return ret; } +unsigned int unaccepted_memory_order(void) +{ + struct efi_unaccepted_memory *unaccepted = efi_get_unaccepted_table(); + + if (unaccepted && unaccepted->unit_size >= PAGE_SIZE) + return ilog2(unaccepted->unit_size >> PAGE_SHIFT); + return 0; +} +EXPORT_SYMBOL_GPL(unaccepted_memory_order); + +int mark_memory_unaccepted(phys_addr_t start, unsigned long size) +{ + struct efi_unaccepted_memory *unaccepted; + unsigned long range_start, range_end; + struct accept_range range, *entry; + phys_addr_t end = start + size; + unsigned long flags; + u64 unit_size; + + if (!arch_has_unaccept_memory()) + return -EOPNOTSUPP; + + unaccepted = efi_get_unaccepted_table(); + if (!unaccepted) + return -EOPNOTSUPP; + + unit_size = unaccepted->unit_size; + if (WARN_ON_ONCE(!IS_ALIGNED(start, unit_size) || + !IS_ALIGNED(size, unit_size))) + return -EINVAL; + + if (WARN_ON_ONCE(start < unaccepted->phys_base || end <= start)) + return -ERANGE; + + start -= unaccepted->phys_base; + end -= unaccepted->phys_base; + + /* Make sure not to overrun the bitmap */ + if (WARN_ON_ONCE(end > unaccepted->size * unit_size * BITS_PER_BYTE)) + return -ERANGE; + + range.start = start / unit_size; + range.end = end / unit_size; +retry: + spin_lock_irqsave(&unaccepted_memory_lock, flags); + + list_for_each_entry(entry, &accepting_list, list) { + if (entry->end <= range.start) + continue; + if (entry->start >= range.end) + continue; + + spin_unlock_irqrestore(&unaccepted_memory_lock, flags); + goto retry; + } + + list_add(&range.list, &accepting_list); + + range_start = range.start; + for_each_clear_bitrange_from(range_start, range_end, + unaccepted->bitmap, range.end) { + unsigned long phys_start, phys_end; + unsigned long len = range_end - range_start; + + phys_start = range_start * unit_size + unaccepted->phys_base; + phys_end = range_end * unit_size + unaccepted->phys_base; + + bitmap_set(unaccepted->bitmap, range_start, len); + spin_unlock(&unaccepted_memory_lock); + + debug_pagealloc_map_pages(pfn_to_page(PHYS_PFN(phys_start)), + (phys_end - phys_start) >> PAGE_SHIFT); + arch_unaccept_memory(phys_start, phys_end); + + spin_lock(&unaccepted_memory_lock); + } + + list_del(&range.list); + + touch_softlockup_watchdog(); + + spin_unlock_irqrestore(&unaccepted_memory_lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(mark_memory_unaccepted); + #ifdef CONFIG_PROC_VMCORE static bool unaccepted_memory_vmcore_pfn_is_ram(struct vmcore_cb *cb, unsigned long pfn) diff --git a/include/linux/mm.h b/include/linux/mm.h index 6882ad400935..27bb40651c85 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5362,6 +5362,8 @@ int set_anon_vma_name(unsigned long addr, unsigned long size, bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size); void accept_memory(phys_addr_t start, unsigned long size); +unsigned int unaccepted_memory_order(void); +int mark_memory_unaccepted(phys_addr_t start, unsigned long size); #else @@ -5375,6 +5377,16 @@ static inline void accept_memory(phys_addr_t start, unsigned long size) { } +static inline unsigned int unaccepted_memory_order(void) +{ + return 0; +} + +static inline int mark_memory_unaccepted(phys_addr_t start, unsigned long size) +{ + return -EOPNOTSUPP; +} + #endif static inline bool pfn_is_unaccepted_memory(unsigned long pfn) -- 2.56.0.307.g5fe7bd680c-goog
