The new x86 memcpy_nt() helper in this series maps to
memcpy_flushcache(), and the ZONE_DEVICE fast path uses that primitive
for constant-sized struct page template copies.
The immediately relevant x86_64 sizeof(struct page) values here are 64,
80, and 96 bytes. memcpy_flushcache() currently inlines only the 4, 8,
and 16-byte cases, so even those constant-sized struct page copies fall
through to __memcpy_flushcache().
Add 32, 48, 64, 80, and 96-byte MOVNTI sequences to the main fixed-size
switch so those constant-sized copies can stay on the inline path.
Factor the larger sequences into movnti_8()/16()/32()/64() helpers so
the switch can reuse them without duplicating the inline assembly.
While the ZONE_DEVICE template-copy path only needs 64/80/96-byte
copies, keep 32-byte and 48-byte copies inline as well rather than
sending those nearby fixed-size cases back to __memcpy_flushcache().
A microbenchmark compared two memcpy_flushcache()-style helpers that
shared the same generic body and differed only in whether
32/48/64/80/96-byte copies stayed in the fixed-size switch or were
redirected to __memcpy_flushcache() when the destination was not 8-byte
aligned. The timed interval covered the copy loop only; wmb() was used
only to separate rounds.
In pseudo-code, the two variants were:
variant A:
switch (len) {
case 32:
case 48:
case 64:
case 80:
case 96:
do fixed-size MOVNTI stores;
return;
default:
generic __memcpy_flushcache()-style body;
}
variant B:
switch (len) {
case 32:
case 48:
case 64:
case 80:
case 96:
if (!IS_ALIGNED(dst, 8))
goto generic_path;
do fixed-size MOVNTI stores;
return;
default:
generic_path:
generic __memcpy_flushcache()-style body;
}
For offsets 1..7 averaged, keeping those sizes in the fixed-size
switch took about 0.410/0.411/0.426/0.426/0.428 us per
32/48/64/80/96-byte copy, while redirecting the same cases to
__memcpy_flushcache() took about 0.962/0.956/0.923/0.998/1.001 us.
Based on that measurement, keep these new cases in the main
memcpy_flushcache() fixed-size switch without a separate
alignment check.
Signed-off-by: Li Zhe <[email protected]>
---
arch/x86/include/asm/string_64.h | 56 +++++++++++++++++++++++++++++++-
1 file changed, 55 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 6cb9e0ac7fa0..1e634264a31b 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -83,7 +83,35 @@ int strcmp(const char *cs, const char *ct);
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
#define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
-static __always_inline void memcpy_flushcache(void *dst, const void *src,
size_t cnt)
+
+static __always_inline void movnti_8(void *dst, const void *src)
+{
+ asm volatile("movntiq %1, %0"
+ : "=m"(*(u64 *)dst)
+ : "r"(*(const u64 *)src)
+ : "memory");
+}
+
+static __always_inline void movnti_16(void *dst, const void *src)
+{
+ movnti_8(dst, src);
+ movnti_8(dst + 8, src + 8);
+}
+
+static __always_inline void movnti_32(void *dst, const void *src)
+{
+ movnti_16(dst, src);
+ movnti_16(dst + 16, src + 16);
+}
+
+static __always_inline void movnti_64(void *dst, const void *src)
+{
+ movnti_32(dst, src);
+ movnti_32(dst + 32, src + 32);
+}
+
+static __always_inline void memcpy_flushcache(void *dst, const void *src,
+ size_t cnt)
{
if (__builtin_constant_p(cnt)) {
switch (cnt) {
@@ -97,8 +125,34 @@ static __always_inline void memcpy_flushcache(void *dst,
const void *src, size_t
asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) :
"r"(*(u64 *)src));
asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8))
: "r"(*(u64 *)(src + 8)));
return;
+ /*
+ * The relevant fixed-size copies here are the
+ * x86_64 struct page sizes: 64, 80, and 96 bytes.
+ * Keep 32-byte and 48-byte copies inline as well
+ * instead of sending those nearby fixed-size
+ * cases back to __memcpy_flushcache().
+ */
+ case 32:
+ movnti_32(dst, src);
+ return;
+ case 48:
+ movnti_32(dst, src);
+ movnti_16(dst + 32, src + 32);
+ return;
+ case 64:
+ movnti_64(dst, src);
+ return;
+ case 80:
+ movnti_64(dst, src);
+ movnti_16(dst + 64, src + 64);
+ return;
+ case 96:
+ movnti_64(dst, src);
+ movnti_32(dst + 64, src + 64);
+ return;
}
}
+
__memcpy_flushcache(dst, src, cnt);
}
--
2.20.1
