On x86_64 architecture, copying cold memory folios using standard store
instructions pollutes CPU caches with cold data, increasing memory
latency and bandwidth overhead.

Support machine-check safe non-temporal copying of folios:
- Add copy_mc_page_nt() to copy a 4KiB page using non-temporal stores,
  prefetching, and MCE exception handling.
- Add copy_mc_highpage_nt() to map and copy a 4KiB page.
- Add folio_mc_copy_nt() to copy a folio using copy_mc_highpage_nt().

Because non-temporal stores are weakly ordered, a wmb() is issued at the
end of folio_mc_copy_nt() before page tables are updated to ensure the
new folio contents are globally visible. In the event of task preemption
and CPU migration mid-folio, the scheduler's locked instructions on the
departing CPU serialize and drain earlier stores, while the final wmb()
covers the remainder on the new CPU.

Suggested-by: Alirad Malek <[email protected]>
Suggested-by: Dimitrios Palyvos-Giannas <[email protected]>
Signed-off-by: Huan Nguyen <[email protected]>
---
 arch/x86/include/asm/page_64.h |  2 ++
 arch/x86/lib/copy_page_64.S    | 65 ++++++++++++++++++++++++++++++++++
 include/linux/highmem.h        | 21 +++++++++++
 include/linux/mm.h             |  1 +
 mm/util.c                      | 29 +++++++++++++++
 5 files changed, 118 insertions(+)

diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 1895c207f629..f37126b56239 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -114,7 +114,9 @@ static inline void clear_page(void *addr)
        clear_pages(addr, 1);
 }
 
+#define __HAVE_ARCH_COPY_PAGE_NT 1
 void copy_page(void *to, void *from);
+unsigned long copy_mc_page_nt(void *to, const void *from);
 KCFI_REFERENCE(copy_page);
 
 /*
diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S
index d8e87fedc20d..185fa3bc34c0 100644
--- a/arch/x86/lib/copy_page_64.S
+++ b/arch/x86/lib/copy_page_64.S
@@ -6,6 +6,7 @@
 #include <linux/cfi_types.h>
 #include <asm/cpufeatures.h>
 #include <asm/alternative.h>
+#include <asm/asm.h>
 
 /*
  * Some CPUs run faster using the string copy instructions (sane microcode).
@@ -88,3 +89,67 @@ SYM_FUNC_START_LOCAL(copy_page_regs)
        addq    $2*8, %rsp
        RET
 SYM_FUNC_END(copy_page_regs)
+
+/*
+ * Copy an entire 4KB page using non-temporal stores (movnti).
+ * Returns 0 on success, or number of bytes not copied on error.
+ */
+SYM_TYPED_FUNC_START(copy_mc_page_nt)
+       subq    $2*8, %rsp
+       movq    %rbx, (%rsp)
+       movq    %r12, 1*8(%rsp)
+
+       xorl    %eax, %eax
+       .p2align 4,,10
+       .p2align 3
+
+.L_nt0:        movq    0x8*0(%rsi,%rax), %rdx
+.L_nt1:        movq    0x8*1(%rsi,%rax), %rcx
+.L_nt2:        movq    0x8*2(%rsi,%rax), %rbx
+.L_nt3:        movq    0x8*3(%rsi,%rax), %r8
+.L_nt4:        movq    0x8*4(%rsi,%rax), %r9
+.L_nt5:        movq    0x8*5(%rsi,%rax), %r10
+.L_nt6:        movq    0x8*6(%rsi,%rax), %r11
+.L_nt7:        movq    0x8*7(%rsi,%rax), %r12
+
+       prefetchnta 5*64(%rsi,%rax)
+
+       movnti  %rdx, 0x8*0(%rdi,%rax)
+       movnti  %rcx, 0x8*1(%rdi,%rax)
+       movnti  %rbx, 0x8*2(%rdi,%rax)
+       movnti  %r8,  0x8*3(%rdi,%rax)
+       movnti  %r9,  0x8*4(%rdi,%rax)
+       movnti  %r10, 0x8*5(%rdi,%rax)
+       movnti  %r11, 0x8*6(%rdi,%rax)
+       movnti  %r12, 0x8*7(%rdi,%rax)
+
+       addq    $64, %rax
+       cmpq    $4096, %rax
+       jne     .L_nt0
+
+       movq    (%rsp), %rbx
+       movq    1*8(%rsp), %r12
+       addq    $2*8, %rsp
+       xorl    %eax, %eax
+       RET
+
+.E_nt:
+       negl    %eax
+       addl    $4096, %eax
+       sfence
+       movq    (%rsp), %rbx
+       movq    1*8(%rsp), %r12
+       addq    $2*8, %rsp
+       RET
+
+       _ASM_EXTABLE_TYPE(.L_nt0, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+       _ASM_EXTABLE_TYPE(.L_nt1, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+       _ASM_EXTABLE_TYPE(.L_nt2, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+       _ASM_EXTABLE_TYPE(.L_nt3, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+       _ASM_EXTABLE_TYPE(.L_nt4, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+       _ASM_EXTABLE_TYPE(.L_nt5, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+       _ASM_EXTABLE_TYPE(.L_nt6, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+       _ASM_EXTABLE_TYPE(.L_nt7, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE)
+
+SYM_FUNC_END(copy_mc_page_nt)
+EXPORT_SYMBOL_GPL(copy_mc_page_nt)
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index d7aac9de1c8a..264b3f109151 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -484,6 +484,27 @@ static inline int copy_mc_highpage(struct page *to, struct 
page *from)
 }
 #endif
 
+#ifdef __HAVE_ARCH_COPY_PAGE_NT
+static inline int copy_mc_highpage_nt(struct page *to, struct page *from)
+{
+       unsigned long ret;
+       char *vfrom, *vto;
+
+       vfrom = kmap_local_page(from);
+       vto = kmap_local_page(to);
+       ret = copy_mc_page_nt(vto, vfrom);
+       if (!ret)
+               kmsan_copy_page_meta(to, from);
+       kunmap_local(vto);
+       kunmap_local(vfrom);
+
+       if (ret)
+               memory_failure_queue(page_to_pfn(from), 0);
+
+       return ret;
+}
+#endif
+
 static inline void memcpy_page(struct page *dst_page, size_t dst_off,
                               struct page *src_page, size_t src_off,
                               size_t len)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 9f167950926a..549bb21027e4 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2127,6 +2127,7 @@ void __folio_put(struct folio *folio);
 void split_page(struct page *page, unsigned int order);
 void folio_copy(struct folio *dst, struct folio *src);
 int folio_mc_copy(struct folio *dst, struct folio *src);
+int folio_mc_copy_nt(struct folio *dst, struct folio *src);
 
 unsigned long nr_free_buffer_pages(void);
 
diff --git a/mm/util.c b/mm/util.c
index 8acfe2148ba5..0418529264fd 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -778,6 +778,35 @@ int folio_mc_copy(struct folio *dst, struct folio *src)
 }
 EXPORT_SYMBOL(folio_mc_copy);
 
+int folio_mc_copy_nt(struct folio *dst, struct folio *src)
+{
+       long nr = folio_nr_pages(src);
+       long i = 0;
+
+       for (;;) {
+#ifdef __HAVE_ARCH_COPY_PAGE_NT
+               if (copy_mc_highpage_nt(folio_page(dst, i), folio_page(src, i)))
+                       return -EHWPOISON;
+#else
+               if (copy_mc_highpage(folio_page(dst, i), folio_page(src, i)))
+                       return -EHWPOISON;
+#endif
+               if (++i == nr)
+                       break;
+               cond_resched();
+       }
+
+#ifdef __HAVE_ARCH_COPY_PAGE_NT
+       /*
+        * Drain weakly ordered non-temporal stores before the folio
+        * is published.
+        */
+       wmb();
+#endif
+       return 0;
+}
+EXPORT_SYMBOL(folio_mc_copy_nt);
+
 int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;
 static int sysctl_overcommit_ratio __read_mostly = 50;
 static unsigned long sysctl_overcommit_kbytes __read_mostly;
-- 
2.56.0.398.g882cbad275-goog


Reply via email to