Introduce memcpy_nt() and memcpy_nt_drain() for write-once copy sites that want a named non-temporal copy primitive plus an explicit drain step.
On x86_64, override both helpers in arch/x86/include/asm/string_64.h using the usual self-macro pattern, next to the existing memcpy_flushcache() backend that memcpy_nt() wraps. The x86_64 implementation maps memcpy_nt() to memcpy_flushcache() and uses wmb() for memcpy_nt_drain(), because that backend issues MOVNTI stores and callers need an ordering point before later normal stores that depend on those writes becoming visible. include/linux/string.h provides the generic fallback under memcpy_nt() as plain memcpy() and leaves memcpy_nt_drain() empty, so architectures that do not override memcpy_nt() do not pay an unconditional barrier. Architectures that later grow a specialized memcpy_nt() backend can override memcpy_nt_drain() with whatever drain primitive their memory-ordering rules require. The immediate user is the ZONE_DEVICE template-copy path. It populates struct page descriptors in a write-once pattern, so a regular cached memcpy() can incur avoidable write-allocate traffic and cache pollution for data with little near-term reuse. Signed-off-by: Li Zhe <[email protected]> --- arch/x86/include/asm/string_64.h | 22 ++++++++++++++++++++++ include/linux/string.h | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 4635616863f5..6cb9e0ac7fa0 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -4,6 +4,7 @@ #ifdef __KERNEL__ #include <linux/jump_label.h> +#include <asm/barrier.h> /* Written 2002 by Andi Kleen */ @@ -100,6 +101,27 @@ static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t } __memcpy_flushcache(dst, src, cnt); } + +#define memcpy_nt memcpy_nt +/* + * Reuse the existing x86 flushcache backend as the nt copy primitive. + * Callers pair it with memcpy_nt_drain() when later stores must be + * ordered after the copy. + */ +static __always_inline void memcpy_nt(void *dst, const void *src, size_t cnt) +{ + memcpy_flushcache(dst, src, cnt); +} + +#define memcpy_nt_drain memcpy_nt_drain +static __always_inline void memcpy_nt_drain(void) +{ + /* + * Order the prior MOVNTI stores issued by memcpy_flushcache() + * before later normal stores. + */ + wmb(); +} #endif #endif /* __KERNEL__ */ diff --git a/include/linux/string.h b/include/linux/string.h index 5702daca4326..a109b2f86ca6 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -278,6 +278,29 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) } #endif +#ifndef memcpy_nt +/* + * memcpy_nt() requests a non-temporal copy when the architecture has a + * suitable backend. Architectures that do not override it fall back to + * memcpy(). + */ +static inline void memcpy_nt(void *dst, const void *src, size_t cnt) +{ + memcpy(dst, src, cnt); +} +#endif + +#ifndef memcpy_nt_drain +/* + * Callers use memcpy_nt_drain() before later normal stores that need to + * be ordered after memcpy_nt(). Architectures without a specialized + * backend can leave it empty. + */ +static inline void memcpy_nt_drain(void) +{ +} +#endif + void *memchr_inv(const void *s, int c, size_t n); char *strreplace(char *str, char old, char new); -- 2.20.1
