__vmap_pages_range_noflush() and friends can install some PTEs before
failing and leave them mapped, and the kernel callers do not agree on
who cleans them up. For example, pcpu_map_pages() and
kmsan_ioremap_page_range() unmap the leftovers themselves, while
vm_module_tags_populate() and the __GFP_NOFAIL retry loop in
__vmalloc_area_node() relied on the mapping functions cleaning up and
did not call anything like vunmap_range() themselves. When the same
range is mapped again, the attempt hits the leftovers and fails, with
BUG() in vmap_pte_range() for huge mappings.
After discussing with Suren and Ulad, we decided the cleanup belongs
to __vmap_pages_range_noflush() and friends, so the callers no longer
need to unmap the partial mappings themselves. Each function now
undoes the PTEs it installed itself.
The rollback calls the low-level __vunmap_range_noflush(), it just
clears the PTEs of the range it is given, which is all a rollback
needs. It cannot use vunmap_range_noflush() because these mapping
functions also map the KMSAN shadow and origin, and for a metadata
range its hook would look up the metadata of the metadata, get 0
and BUG() on addr >= end. The failed mappings were never accessed,
no TLB flush needed.
Fixes: 9376130c390a ("mm/vmalloc: add support for __GFP_NOFAIL")
Fixes: 0f9b685626da ("alloc_tag: populate memory for module tags as needed")
Reported-by: Sashiko <[email protected]>
Cc: [email protected]
Signed-off-by: Hao Ge <[email protected]>
---
mm/kmsan/shadow.c | 4 ++++
mm/vmalloc.c | 31 +++++++++++++++++++++++++++++--
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c
index 0c88d89bf0d6..2166086d3dc3 100644
--- a/mm/kmsan/shadow.c
+++ b/mm/kmsan/shadow.c
@@ -258,6 +258,10 @@ int kmsan_vmap_pages_range_noflush(unsigned long start,
unsigned long end,
o_pages, page_shift);
kmsan_leave_runtime();
if (mapped) {
+ /* Undo the shadow mapping set up above. */
+ kmsan_enter_runtime();
+ __vunmap_range_noflush(shadow_start, shadow_end);
+ kmsan_leave_runtime();
err = mapped;
goto ret;
}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 859e6d2d57a3..9bbf75706627 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -349,6 +349,10 @@ static int vmap_range_noflush(unsigned long addr, unsigned
long end,
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
arch_sync_kernel_mappings(start, end);
+ /* Undo the PTEs installed before the failure. */
+ if (err)
+ __vunmap_range_noflush(start, end);
+
return err;
}
@@ -363,6 +367,9 @@ int vmap_page_range(unsigned long addr, unsigned long end,
if (!err)
err = kmsan_ioremap_page_range(addr, end, phys_addr, prot,
ioremap_max_page_shift);
+ if (err)
+ __vunmap_range_noflush(addr, end);
+
return err;
}
@@ -667,6 +674,10 @@ static int vmap_small_pages_range_noflush(unsigned long
addr, unsigned long end,
if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
arch_sync_kernel_mappings(start, end);
+ /* Undo the PTEs installed before the failure. */
+ if (err)
+ __vunmap_range_noflush(start, end);
+
return err;
}
@@ -683,6 +694,7 @@ int __vmap_pages_range_noflush(unsigned long addr, unsigned
long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
+ unsigned long start = addr;
WARN_ON(page_shift < PAGE_SHIFT);
@@ -696,8 +708,14 @@ int __vmap_pages_range_noflush(unsigned long addr,
unsigned long end,
err = vmap_range_noflush(addr, addr + (1UL << page_shift),
page_to_phys(pages[i]), prot,
page_shift);
- if (err)
+ if (err) {
+ /*
+ * vmap_range_noflush() undoes its own partial work,
+ * discard the chunks that already mapped.
+ */
+ __vunmap_range_noflush(start, addr);
return err;
+ }
addr += 1UL << page_shift;
}
@@ -714,7 +732,16 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned
long end,
if (ret)
return ret;
- return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+
+ ret = __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+ /*
+ * The page tables undo themselves on failure. Tear down the
+ * metadata that was fully set up before the mapping failed.
+ */
+ if (ret)
+ kmsan_vunmap_range_noflush(addr, end);
+
+ return ret;
}
static int __vmap_pages_range(unsigned long addr, unsigned long end,
--
2.25.1
