The branch main has been updated by fuz: URL: https://cgit.FreeBSD.org/src/commit/?id=d8385768fb12e6205d73a20ad05fba9f3281b6e1
commit d8385768fb12e6205d73a20ad05fba9f3281b6e1 Author: Robert Clausecker <[email protected]> AuthorDate: 2023-08-03 22:48:32 +0000 Commit: Robert Clausecker <[email protected]> CommitDate: 2023-08-03 22:54:23 +0000 lib/libc/amd64/string/strlen.S: add amd64 baseline kernel This performs very well. x86-64-v3 and x86-64-v4 kernels were written, too, but performed worse than the baseline kernel on short strings. These may be added at a future point in time if the performance issues can be fixed. os: FreeBSD arch: amd64 cpu: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz │ strlen_scalar.out │ strlen_baseline.out │ │ B/s │ B/s vs base │ Short 1.667Gi ± 1% 2.676Gi ± 1% +60.55% (p=0.000 n=20) Mid 5.459Gi ± 1% 8.756Gi ± 1% +60.39% (p=0.000 n=20) Long 15.34Gi ± 0% 52.27Gi ± 0% +240.64% (p=0.000 n=20) geomean 5.188Gi 10.70Gi +106.24% Sponsored by: The FreeBSD Foundation Approved by: kib Reviewed by: mjg jrtc27 Differential Revision: https://reviews.freebsd.org/D40693 --- lib/libc/amd64/string/strlen.S | 58 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/lib/libc/amd64/string/strlen.S b/lib/libc/amd64/string/strlen.S index 1d2428e3420e..7e2514de44b0 100644 --- a/lib/libc/amd64/string/strlen.S +++ b/lib/libc/amd64/string/strlen.S @@ -1,11 +1,18 @@ -/* +/*- * Written by Mateusz Guzik <[email protected]> + * Copyright (c) 2023 The FreeBSD Foundation + * + * Portions of this software were developed by Robert Clausecker + * <[email protected]> under sponsorship from the FreeBSD Foundation. + * * Public domain. */ #include <machine/asm.h> __FBSDID("$FreeBSD$"); +#include "amd64_archlevel.h" + /* * Note: this routine was written with kernel use in mind (read: no simd), * it is only present in userspace as a temporary measure until something @@ -14,6 +21,11 @@ __FBSDID("$FreeBSD$"); #define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ +ARCHFUNCS(strlen) + ARCHFUNC(strlen, scalar) + ARCHFUNC(strlen, baseline) +ENDARCHFUNCS(strlen) + /* * strlen(string) * %rdi @@ -30,7 +42,7 @@ __FBSDID("$FreeBSD$"); * * The latter contains a 32-bit variant of the same algorithm coded in assembly for i386. */ -ENTRY(strlen) +ARCHENTRY(strlen, scalar) movabsq $0xfefefefefefefeff,%r8 movabsq $0x8080808080808080,%r9 @@ -76,6 +88,46 @@ ENTRY(strlen) leaq (%rcx,%rdi),%rax subq %r10,%rax ret -END(strlen) +ARCHEND(strlen, scalar) + +ARCHENTRY(strlen, baseline) + mov %rdi, %rcx + pxor %xmm1, %xmm1 + and $~0xf, %rdi # align string + pcmpeqb (%rdi), %xmm1 # compare head (with junk before string) + mov %rcx, %rsi # string pointer copy for later + and $0xf, %ecx # amount of bytes rdi is past 16 byte alignment + pmovmskb %xmm1, %eax + add $32, %rdi # advance to next iteration + shr %cl, %eax # clear out matches in junk bytes + test %eax, %eax # any match? (can't use ZF from SHR as CL=0 is possible) + jnz 2f + + ALIGN_TEXT +1: pxor %xmm1, %xmm1 + pcmpeqb -16(%rdi), %xmm1 # find NUL bytes + pmovmskb %xmm1, %eax + test %eax, %eax # were any NUL bytes present? + jnz 3f + + /* the same unrolled once more */ + pxor %xmm1, %xmm1 + pcmpeqb (%rdi), %xmm1 + pmovmskb %xmm1, %eax + add $32, %rdi # advance to next iteration + test %eax, %eax + jz 1b + + /* match found in loop body */ + sub $16, %rdi # undo half the advancement +3: tzcnt %eax, %eax # find the first NUL byte + sub %rsi, %rdi # string length until beginning of (%rdi) + lea -16(%rdi, %rax, 1), %rax # that plus loc. of NUL byte: full string length + ret + + /* match found in head */ +2: tzcnt %eax, %eax # compute string length + ret +ARCHEND(strlen, baseline) .section .note.GNU-stack,"",%progbits
