Device DAX maps its memory with vmf_insert_folio_pud(), which takes a
folio reference, adds a PUD rmap and accounts the pages to the mm.
zap_huge_pud() drops all three, but copy_huge_pud() copies the PUD into
the child without taking any of them. Each child's unmap then drops
what the parent took:

- the folio is a reference short, so with two children its refcount
  drops to zero and free_zone_device_folio() runs while the parent
  still maps it;
- the child's file counter goes negative, "Bad rss-counter state" when
  its mm is freed;
- the parent's unmap drives the entire mapcount below zero.

fork copies the page tables of a device DAX mapping that has a guard
region, and commit f807123d578d ("mm: allow guard regions in file-backed
and read-only mappings") allowed MADV_GUARD_INSTALL there.

Take the reference, the rmap and the counter for a PUD that
vm_normal_page_pud() resolves to a page, as copy_present_ptes() does for
a PTE, with a new folio_dup_file_rmap_pud() helper. copy_huge_pud() now
needs both VMAs, so pass only them and take the mms from them.

Fixes: aed877c2b425 ("device/dax: properly refcount device dax pages when 
mapping")
Suggested-by: David Hildenbrand (Arm) <[email protected]>
Cc: [email protected]
Signed-off-by: Konstantin Khorenko <[email protected]>
---
 include/linux/huge_mm.h |  5 ++---
 include/linux/rmap.h    | 22 ++++++++++++++++++++++
 mm/huge_memory.c        | 27 +++++++++++++++++++--------
 mm/memory.c             |  4 +---
 4 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index c745f7ad22987..fac05120ab774 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -12,9 +12,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct 
*src_mm,
                  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
                  struct vm_area_struct *dst_vma, struct vm_area_struct 
*src_vma);
 bool huge_pmd_set_accessed(struct vm_fault *vmf);
-int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-                 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
-                 struct vm_area_struct *vma);
+int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct 
*src_vma,
+               pud_t *dst_pud, pud_t *src_pud, unsigned long addr);
 
 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 74cca0e3c7264..52fff8dc74213 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -564,6 +564,28 @@ static inline void folio_dup_file_rmap_pmd(struct folio 
*folio,
 #endif
 }
 
+/**
+ * folio_dup_file_rmap_pud - duplicate a PUD mapping of a page range of a folio
+ * @folio:     The folio to duplicate the mapping of
+ * @page:      The first page to duplicate the mapping of
+ * @dst_vma:   The destination vm area
+ *
+ * The page range of the folio is defined by [page, page + HPAGE_PUD_NR)
+ *
+ * The caller needs to hold the page table lock.
+ */
+static inline void folio_dup_file_rmap_pud(struct folio *folio,
+               struct page *page, struct vm_area_struct *dst_vma)
+{
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
+       defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
+       __folio_dup_file_rmap(folio, page, HPAGE_PUD_NR, dst_vma,
+                             PGTABLE_LEVEL_PUD);
+#else
+       WARN_ON_ONCE(true);
+#endif
+}
+
 static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio,
                struct page *page, int nr_pages, struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma, enum pgtable_level level)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 14599d0fff013..30c21e85d21bf 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2072,11 +2072,13 @@ void touch_pud(struct vm_area_struct *vma, unsigned 
long addr,
                update_mmu_cache_pud(vma, addr, pud);
 }
 
-int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
-                 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
-                 struct vm_area_struct *vma)
+int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct 
*src_vma,
+               pud_t *dst_pud, pud_t *src_pud, unsigned long addr)
 {
+       struct mm_struct *dst_mm = dst_vma->vm_mm;
+       struct mm_struct *src_mm = src_vma->vm_mm;
        spinlock_t *dst_ptl, *src_ptl;
+       struct page *page;
        pud_t pud;
        int ret;
 
@@ -2089,11 +2091,20 @@ int copy_huge_pud(struct mm_struct *dst_mm, struct 
mm_struct *src_mm,
        if (unlikely(!pud_trans_huge(pud)))
                goto out_unlock;
 
-       /*
-        * TODO: once we support anonymous pages, use
-        * folio_try_dup_anon_rmap_*() and split if duplicating fails.
-        */
-       if (vma_is_cow_mapping(vma) && pud_write(pud)) {
+       page = vm_normal_page_pud(src_vma, addr, pud);
+       if (page) {
+               struct folio *folio = page_folio(page);
+
+               folio_get(folio);
+               /*
+                * TODO: once we support anonymous pages, use
+                * folio_try_dup_anon_rmap_*() and split if duplicating fails.
+                */
+               folio_dup_file_rmap_pud(folio, page, dst_vma);
+               add_mm_counter(dst_mm, mm_counter_file(folio), HPAGE_PUD_NR);
+       }
+
+       if (vma_is_cow_mapping(src_vma) && pud_write(pud)) {
                pudp_set_wrprotect(src_mm, addr, src_pud);
                pud = pud_wrprotect(pud);
        }
diff --git a/mm/memory.c b/mm/memory.c
index b65b56fe97148..66fbe8ed5bac0 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1463,7 +1463,6 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct 
vm_area_struct *src_vma,
               unsigned long end)
 {
        struct mm_struct *dst_mm = dst_vma->vm_mm;
-       struct mm_struct *src_mm = src_vma->vm_mm;
        pud_t *src_pud, *dst_pud;
        unsigned long next;
 
@@ -1477,8 +1476,7 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct 
vm_area_struct *src_vma,
                        int err;
 
                        VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
-                       err = copy_huge_pud(dst_mm, src_mm,
-                                           dst_pud, src_pud, addr, src_vma);
+                       err = copy_huge_pud(dst_vma, src_vma, dst_pud, src_pud, 
addr);
                        if (err == -ENOMEM)
                                return -ENOMEM;
                        if (!err)

-- 
2.43.0


Reply via email to