On x86_64 architecture, copying cold memory folios using standard store instructions pollutes CPU caches with cold data, increasing memory latency and bandwidth overhead.
Support machine-check safe non-temporal copying of folios: - Add copy_mc_page_nt() to copy a 4KiB page using non-temporal stores, prefetching, and MCE exception handling. - Add copy_mc_highpage_nt() to map and copy a 4KiB page. - Add folio_mc_copy_nt() to copy a folio using copy_mc_highpage_nt(). Because non-temporal stores are weakly ordered, a wmb() is issued at the end of folio_mc_copy_nt() before page tables are updated to ensure the new folio contents are globally visible. In the event of task preemption and CPU migration mid-folio, the scheduler's locked instructions on the departing CPU serialize and drain earlier stores, while the final wmb() covers the remainder on the new CPU. Suggested-by: Alirad Malek <[email protected]> Suggested-by: Dimitrios Palyvos-Giannas <[email protected]> Signed-off-by: Huan Nguyen <[email protected]> --- arch/x86/include/asm/page_64.h | 2 ++ arch/x86/lib/copy_page_64.S | 65 ++++++++++++++++++++++++++++++++++ include/linux/highmem.h | 21 +++++++++++ include/linux/mm.h | 1 + mm/util.c | 29 +++++++++++++++ 5 files changed, 118 insertions(+) diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h index 1895c207f629..f37126b56239 100644 --- a/arch/x86/include/asm/page_64.h +++ b/arch/x86/include/asm/page_64.h @@ -114,7 +114,9 @@ static inline void clear_page(void *addr) clear_pages(addr, 1); } +#define __HAVE_ARCH_COPY_PAGE_NT 1 void copy_page(void *to, void *from); +unsigned long copy_mc_page_nt(void *to, const void *from); KCFI_REFERENCE(copy_page); /* diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S index d8e87fedc20d..185fa3bc34c0 100644 --- a/arch/x86/lib/copy_page_64.S +++ b/arch/x86/lib/copy_page_64.S @@ -6,6 +6,7 @@ #include <linux/cfi_types.h> #include <asm/cpufeatures.h> #include <asm/alternative.h> +#include <asm/asm.h> /* * Some CPUs run faster using the string copy instructions (sane microcode). @@ -88,3 +89,67 @@ SYM_FUNC_START_LOCAL(copy_page_regs) addq $2*8, %rsp RET SYM_FUNC_END(copy_page_regs) + +/* + * Copy an entire 4KB page using non-temporal stores (movnti). + * Returns 0 on success, or number of bytes not copied on error. + */ +SYM_TYPED_FUNC_START(copy_mc_page_nt) + subq $2*8, %rsp + movq %rbx, (%rsp) + movq %r12, 1*8(%rsp) + + xorl %eax, %eax + .p2align 4,,10 + .p2align 3 + +.L_nt0: movq 0x8*0(%rsi,%rax), %rdx +.L_nt1: movq 0x8*1(%rsi,%rax), %rcx +.L_nt2: movq 0x8*2(%rsi,%rax), %rbx +.L_nt3: movq 0x8*3(%rsi,%rax), %r8 +.L_nt4: movq 0x8*4(%rsi,%rax), %r9 +.L_nt5: movq 0x8*5(%rsi,%rax), %r10 +.L_nt6: movq 0x8*6(%rsi,%rax), %r11 +.L_nt7: movq 0x8*7(%rsi,%rax), %r12 + + prefetchnta 5*64(%rsi,%rax) + + movnti %rdx, 0x8*0(%rdi,%rax) + movnti %rcx, 0x8*1(%rdi,%rax) + movnti %rbx, 0x8*2(%rdi,%rax) + movnti %r8, 0x8*3(%rdi,%rax) + movnti %r9, 0x8*4(%rdi,%rax) + movnti %r10, 0x8*5(%rdi,%rax) + movnti %r11, 0x8*6(%rdi,%rax) + movnti %r12, 0x8*7(%rdi,%rax) + + addq $64, %rax + cmpq $4096, %rax + jne .L_nt0 + + movq (%rsp), %rbx + movq 1*8(%rsp), %r12 + addq $2*8, %rsp + xorl %eax, %eax + RET + +.E_nt: + negl %eax + addl $4096, %eax + sfence + movq (%rsp), %rbx + movq 1*8(%rsp), %r12 + addq $2*8, %rsp + RET + + _ASM_EXTABLE_TYPE(.L_nt0, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_nt1, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_nt2, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_nt3, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_nt4, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_nt5, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_nt6, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + _ASM_EXTABLE_TYPE(.L_nt7, .E_nt, EX_TYPE_DEFAULT_MCE_SAFE) + +SYM_FUNC_END(copy_mc_page_nt) +EXPORT_SYMBOL_GPL(copy_mc_page_nt) diff --git a/include/linux/highmem.h b/include/linux/highmem.h index d7aac9de1c8a..264b3f109151 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -484,6 +484,27 @@ static inline int copy_mc_highpage(struct page *to, struct page *from) } #endif +#ifdef __HAVE_ARCH_COPY_PAGE_NT +static inline int copy_mc_highpage_nt(struct page *to, struct page *from) +{ + unsigned long ret; + char *vfrom, *vto; + + vfrom = kmap_local_page(from); + vto = kmap_local_page(to); + ret = copy_mc_page_nt(vto, vfrom); + if (!ret) + kmsan_copy_page_meta(to, from); + kunmap_local(vto); + kunmap_local(vfrom); + + if (ret) + memory_failure_queue(page_to_pfn(from), 0); + + return ret; +} +#endif + static inline void memcpy_page(struct page *dst_page, size_t dst_off, struct page *src_page, size_t src_off, size_t len) diff --git a/include/linux/mm.h b/include/linux/mm.h index 9f167950926a..549bb21027e4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2127,6 +2127,7 @@ void __folio_put(struct folio *folio); void split_page(struct page *page, unsigned int order); void folio_copy(struct folio *dst, struct folio *src); int folio_mc_copy(struct folio *dst, struct folio *src); +int folio_mc_copy_nt(struct folio *dst, struct folio *src); unsigned long nr_free_buffer_pages(void); diff --git a/mm/util.c b/mm/util.c index 8acfe2148ba5..0418529264fd 100644 --- a/mm/util.c +++ b/mm/util.c @@ -778,6 +778,35 @@ int folio_mc_copy(struct folio *dst, struct folio *src) } EXPORT_SYMBOL(folio_mc_copy); +int folio_mc_copy_nt(struct folio *dst, struct folio *src) +{ + long nr = folio_nr_pages(src); + long i = 0; + + for (;;) { +#ifdef __HAVE_ARCH_COPY_PAGE_NT + if (copy_mc_highpage_nt(folio_page(dst, i), folio_page(src, i))) + return -EHWPOISON; +#else + if (copy_mc_highpage(folio_page(dst, i), folio_page(src, i))) + return -EHWPOISON; +#endif + if (++i == nr) + break; + cond_resched(); + } + +#ifdef __HAVE_ARCH_COPY_PAGE_NT + /* + * Drain weakly ordered non-temporal stores before the folio + * is published. + */ + wmb(); +#endif + return 0; +} +EXPORT_SYMBOL(folio_mc_copy_nt); + int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS; static int sysctl_overcommit_ratio __read_mostly = 50; static unsigned long sysctl_overcommit_kbytes __read_mostly; -- 2.56.0.398.g882cbad275-goog
