Hi Kyrill, > -----Original Message----- > From: [email protected] <[email protected]> > Sent: 06 October 2026 15:15 > To: [email protected] > Cc: Kyrylo Tkachov <[email protected]>; Tamar Christina > <[email protected]> > Subject: [PATCH] aarch64: Use SVE2 ADALP for widening sums > > From: Kyrylo Tkachov <[email protected]> > > The 64-bit accumulator path in precompute_partition_info_sums_ from > Xiph FLAC adds the absolute values of 32-bit residuals. > The relevant statement in src/libFLAC/stream_encoder.c is: > > abs_residual_partition_sum64 += abs (residual[residual_sample]); > > The vectorizer recognizes this as a widening sum. The SVE2 > reduc_widen_ssum expander then emits SADDWB followed by SADDWT. Both > instructions add one half of the narrow vector to the same accumulator. > SADALP performs the same pairwise widening accumulation in one > instruction. The unsigned widening sum has the same issue with UADDWB > and UADDWT. > > Emit the existing predicated SADALP and UADALP patterns from the SVE2 > widening-sum expanders. Use an all-true predicate because the optab is > unpredicated. Keep the existing SVE2.1 and SME2 H-to-S dot-product case. > > With: > > -O3 -mcpu=neoverse-v2 -mautovec-preference=sve-only > -msve-vector-bits=scalable > --param aarch64-vect-unroll-limit=1 > --param vect-partial-vector-usage=0 > > the steady loop for the FLAC reduction changes from: > > .L4: > ld1w z30.s, p7/z, [x0, x2, lsl 2] > mov x4, x2 > add x2, x2, x5 > abs z30.s, p7/m, z30.s > saddwb z31.d, z31.d, z30.s > saddwt z31.d, z31.d, z30.s > cmp w3, w2 > bcs .L4 > > to: > > .L4: > ld1w z30.s, p7/z, [x0, x2, lsl 2] > mov x4, x2 > add x2, x2, x5 > abs z30.s, p7/m, z30.s > sadalp z31.d, p7/m, z30.s > cmp w3, w2 > bcs .L4 > > Bootstrapped and tested on aarch64-none-linux-gnu. > Ok for trunk?
OK. Thanks, Tamar > Thanks, > Kyrill > > Cc: Tamar Christina <[email protected]> > > gcc/ChangeLog: > > * config/aarch64/aarch64-sve2.md > (reduc_widen_ssum<mode><Vnarrow>3): Emit cond_sadalp. > (reduc_widen_usum<mode><Vnarrow>3): Emit cond_uadalp. > (cond_<sve_int_op><mode>, *cond_<sve_int_op><mode>_2, > *cond_<sve_int_op><mode>_z): Fix comments. > > gcc/testsuite/ChangeLog: > > * gcc.target/aarch64/sve2/pr122069_1.c: Expect UADALP. > * gcc.target/aarch64/sve2/widen-sum-adalp-1.c: New test. > * gcc.target/aarch64/sve2/widen-sum-adalp-run.c: Likewise. > > Signed-off-by: Kyrylo Tkachov <[email protected]> > --- > gcc/config/aarch64/aarch64-sve2.md | 47 +++---- > .../gcc.target/aarch64/sve2/pr122069_1.c | 3 +- > .../aarch64/sve2/widen-sum-adalp-1.c | 38 ++++++ > .../aarch64/sve2/widen-sum-adalp-run.c | 123 ++++++++++++++++++ > 4 files changed, 183 insertions(+), 28 deletions(-) > create mode 100644 gcc/testsuite/gcc.target/aarch64/sve2/widen-sum- > adalp-1.c > create mode 100644 gcc/testsuite/gcc.target/aarch64/sve2/widen-sum- > adalp-run.c > > diff --git a/gcc/config/aarch64/aarch64-sve2.md > b/gcc/config/aarch64/aarch64-sve2.md > index c33df283cc2..7d8504db304 100644 > --- a/gcc/config/aarch64/aarch64-sve2.md > +++ b/gcc/config/aarch64/aarch64-sve2.md > @@ -2563,18 +2563,11 @@ > [(set_attr "sve_type" "sve_int_general")] > ) > > -;; Define single step widening for reduc_widen_ssum using SADDWB and > SADDWT > +;; Define single step widening for reduc_widen_ssum using SADALP. > (define_expand "reduc_widen_ssum<mode><Vnarrow>3" > - [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") > - (unspec:SVE_FULL_HSDI > - [(match_operand:SVE_FULL_HSDI 2 "register_operand") > - (match_operand:<VNARROW> 1 "register_operand")] > - UNSPEC_SADDWB)) > - (set (match_dup 0) > - (unspec:SVE_FULL_HSDI > - [(match_dup 0) > - (match_dup 1)] > - UNSPEC_SADDWT))] > + [(match_operand:SVE_FULL_HSDI 0 "register_operand") > + (match_operand:<VNARROW> 1 "register_operand") > + (match_operand:SVE_FULL_HSDI 2 "register_operand")] > "TARGET_SVE2" > { > /* Use dot product to perform double widening sum reductions by > @@ -2588,20 +2581,18 @@ > ones, operands[2])); > DONE; > } > + > + rtx pred = aarch64_ptrue_reg (<VPRED>mode); > + emit_insn (gen_cond_sadalp<mode> (operands[0], pred, operands[2], > + operands[1], operands[2])); > + DONE; > }) > > -;; Define single step widening for reduc_widen_usum using UADDWB and > UADDWT > +;; Define single step widening for reduc_widen_usum using UADALP. > (define_expand "reduc_widen_usum<mode><Vnarrow>3" > - [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w") > - (unspec:SVE_FULL_HSDI > - [(match_operand:SVE_FULL_HSDI 2 "register_operand" "w") > - (match_operand:<VNARROW> 1 "register_operand" "w")] > - UNSPEC_UADDWB)) > - (set (match_dup 0) > - (unspec:SVE_FULL_HSDI > - [(match_dup 0) > - (match_dup 1)] > - UNSPEC_UADDWT))] > + [(match_operand:SVE_FULL_HSDI 0 "register_operand") > + (match_operand:<VNARROW> 1 "register_operand") > + (match_operand:SVE_FULL_HSDI 2 "register_operand")] > "TARGET_SVE2" > { > if (TARGET_SVE2p1_OR_SME2 > @@ -2613,6 +2604,11 @@ > ones, operands[2])); > DONE; > } > + > + rtx pred = aarch64_ptrue_reg (<VPRED>mode); > + emit_insn (gen_cond_uadalp<mode> (operands[0], pred, operands[2], > + operands[1], operands[2])); > + DONE; > }) > > ;; ------------------------------------------------------------------------- > @@ -3344,7 +3340,7 @@ > ;; - UADALP > ;; ------------------------------------------------------------------------- > > -;; Predicated pairwise absolute difference and accumulate with merging. > +;; Predicated add long pairwise and accumulate with merging. > (define_expand "@cond_<sve_int_op><mode>" > [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") > (unspec:SVE_FULL_HSDI > @@ -3364,8 +3360,7 @@ > || CONSTANT_P (operands[4])); > }) > > -;; Predicated pairwise absolute difference and accumulate, merging with > -;; the first input. > +;; Predicated add long pairwise and accumulate, merging with the first input. > (define_insn_and_rewrite "*cond_<sve_int_op><mode>_2" > [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") > (unspec:SVE_FULL_HSDI > @@ -3389,7 +3384,7 @@ > [(set_attr "sve_type" "sve_int_accum")] > ) > > -;; Predicated pairwise absolute difference and accumulate, merging with zero. > +;; Predicated add long pairwise and accumulate, merging with zero. > (define_insn_and_rewrite "*cond_<sve_int_op><mode>_z" > [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") > (unspec:SVE_FULL_HSDI > diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c > b/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c > index 6a347072ae8..02e38f70b54 100644 > --- a/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c > +++ b/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c > @@ -24,8 +24,7 @@ int foo_int(unsigned char *x, unsigned char * restrict y) { > ** foo2_int: > ** ... > ** add z[0-9]+.h, z[0-9]+.h, z[0-9]+.h > -** uaddwb z[0-9]+.s, z[0-9]+.s, z[0-9]+.h > -** uaddwt z[0-9]+.s, z[0-9]+.s, z[0-9]+.h > +** uadalp z[0-9]+.s, p[0-7]/m, z[0-9]+.h > ** ... > */ > int foo2_int(unsigned short *x, unsigned short * restrict y) { > diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c > b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c > new file mode 100644 > index 00000000000..172d6ba645f > --- /dev/null > +++ b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c > @@ -0,0 +1,38 @@ > +/* { dg-do compile } */ > +/* { dg-options "-O3 -march=armv9-a+sve2 -mautovec-preference=sve-only > -msve-vector-bits=scalable --param aarch64-vect-unroll-limit=1 --param vect- > partial-vector-usage=0" } */ > + > +#define DEF(NAME, INPUT, OUTPUT) \ > + OUTPUT \ > + NAME (const INPUT *a, unsigned int n) \ > + { \ > + OUTPUT sum = 0; \ > + for (unsigned int i = 0; i < n; ++i) \ > + sum += a[i]; \ > + return sum; \ > + } > + > +DEF (sum_s8_s16, signed char, short) > +DEF (sum_u8_u16, unsigned char, unsigned short) > +DEF (sum_s16_s32, short, int) > +DEF (sum_u16_u32, unsigned short, unsigned int) > +DEF (sum_s32_s64, int, long long) > +DEF (sum_u32_u64, unsigned int, unsigned long long) > + > +unsigned long long > +sum_abs_s32_u64 (const int *a, unsigned int n) > +{ > + unsigned long long sum = 0; > + for (unsigned int i = 0; i < n; ++i) > + sum += __builtin_abs (a[i]); > + return sum; > +} > + > +/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.h, p[0-7]/m, z[0- > 9]+\.b\n} 1 } } */ > +/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.h, p[0-7]/m, z[0- > 9]+\.b\n} 1 } } */ > +/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.s, p[0-7]/m, z[0- > 9]+\.h\n} 1 } } */ > +/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.s, p[0-7]/m, z[0- > 9]+\.h\n} 1 } } */ > +/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.d, p[0-7]/m, z[0- > 9]+\.s\n} 2 } } */ > +/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.d, p[0-7]/m, z[0- > 9]+\.s\n} 1 } } */ > +/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */ > +/* { dg-final { scan-assembler-not {\tsaddw[bt]\t} } } */ > +/* { dg-final { scan-assembler-not {\tuaddw[bt]\t} } } */ > diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c > b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c > new file mode 100644 > index 00000000000..d064ed99d0c > --- /dev/null > +++ b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c > @@ -0,0 +1,123 @@ > +/* { dg-do run { target aarch64_sve2_hw } } */ > +/* { dg-options "-O3 -march=armv9-a+sve2 -mautovec-preference=sve-only > -msve-vector-bits=scalable --param aarch64-vect-unroll-limit=1 --param vect- > partial-vector-usage=0" } */ > + > +#include <arm_sve.h> > +#include <limits.h> > +#include <stdlib.h> > + > +#define DEF_SUMS(NAME, INPUT, OUTPUT) \ > + __attribute__((noipa)) OUTPUT \ > + NAME (const INPUT *a, unsigned int n) \ > + { \ > + OUTPUT sum = 0; \ > + for (unsigned int i = 0; i < n; ++i) \ > + sum += a[i]; \ > + return sum; \ > + } \ > + \ > + __attribute__((noipa, optimize ("no-tree-vectorize"))) OUTPUT \ > + ref_##NAME (const INPUT *a, unsigned int n) \ > + { \ > + OUTPUT sum = 0; \ > + for (unsigned int i = 0; i < n; ++i) \ > + sum += a[i]; \ > + return sum; \ > + } > + > +DEF_SUMS (sum_s8_s16, signed char, short) > +DEF_SUMS (sum_u8_u16, unsigned char, unsigned short) > +DEF_SUMS (sum_s16_s32, short, int) > +DEF_SUMS (sum_u16_u32, unsigned short, unsigned int) > +DEF_SUMS (sum_s32_s64, int, long long) > +DEF_SUMS (sum_u32_u64, unsigned int, unsigned long long) > + > +__attribute__((noipa)) unsigned long long > +sum_abs_s32_u64 (const int *a, unsigned int n) > +{ > + unsigned long long sum = 0; > + for (unsigned int i = 0; i < n; ++i) > + sum += __builtin_abs (a[i]); > + return sum; > +} > + > +__attribute__((noipa, optimize ("no-tree-vectorize"))) unsigned long long > +ref_abs_s32_u64 (const int *a, unsigned int n) > +{ > + unsigned long long sum = 0; > + for (unsigned int i = 0; i < n; ++i) > + sum += __builtin_abs (a[i]); > + return sum; > +} > + > +int > +main (void) > +{ > + unsigned int vl_b = svcntb (); > + unsigned int vl_h = svcnth (); > + unsigned int vl_w = svcntw (); > + unsigned int count = 4 * vl_b + 3; > + unsigned int allocation_count = count + 1; > + signed char *signed_char_values > + = (signed char *) __builtin_alloca (allocation_count > + * sizeof (signed char)); > + unsigned char *unsigned_char_values > + = (unsigned char *) __builtin_alloca (allocation_count > + * sizeof (unsigned char)); > + short *signed_short_values > + = (short *) __builtin_alloca (allocation_count * sizeof (short)); > + unsigned short *unsigned_short_values > + = (unsigned short *) __builtin_alloca (allocation_count > + * sizeof (unsigned short)); > + int *signed_int_values > + = (int *) __builtin_alloca (allocation_count * sizeof (int)); > + unsigned int *unsigned_int_values > + = (unsigned int *) __builtin_alloca (allocation_count > + * sizeof (unsigned int)); > + > + for (unsigned int i = 0; i < allocation_count; ++i) > + { > + static const signed char signed_char_edges[] = > + { SCHAR_MIN, SCHAR_MAX, 1, 0, -1 }; > + static const unsigned char unsigned_char_edges[] = > + { 0, 1, UCHAR_MAX, UCHAR_MAX - 1, 0x80 }; > + static const short signed_short_edges[] = > + { SHRT_MIN, SHRT_MAX, 1, 0, -1 }; > + static const unsigned short unsigned_short_edges[] = > + { 0, 1, USHRT_MAX, USHRT_MAX - 1, 0x8000 }; > + static const int signed_int_edges[] = > + { INT_MIN + 1, INT_MAX, -1, 0, 1, -123456789, 123456789 }; > + static const unsigned int unsigned_int_edges[] = > + { 0, 1, UINT_MAX, UINT_MAX - 1, 0x80000000U, 123456789U }; > + signed_char_values[i] = signed_char_edges[i % 5]; > + unsigned_char_values[i] = unsigned_char_edges[i % 5]; > + signed_short_values[i] = signed_short_edges[i % 5]; > + unsigned_short_values[i] = unsigned_short_edges[i % 5]; > + signed_int_values[i] = signed_int_edges[i % 7]; > + unsigned_int_values[i] = unsigned_int_edges[i % 6]; > + } > + > + unsigned int lengths[] = > + { 0, 1, vl_w - 1, vl_w, vl_w + 1, vl_h - 1, vl_h, vl_h + 1, > + vl_b - 1, vl_b, vl_b + 1, 2 * vl_b + 1, count }; > + for (unsigned int i = 0; i < sizeof (lengths) / sizeof (lengths[0]); ++i) > + for (unsigned int offset = 0; offset <= 1; ++offset) > + { > + unsigned int n = lengths[i]; > + if ((sum_s8_s16 (signed_char_values + offset, n) > + != ref_sum_s8_s16 (signed_char_values + offset, n)) > + || (sum_u8_u16 (unsigned_char_values + offset, n) > + != ref_sum_u8_u16 (unsigned_char_values + offset, n)) > + || (sum_s16_s32 (signed_short_values + offset, n) > + != ref_sum_s16_s32 (signed_short_values + offset, n)) > + || (sum_u16_u32 (unsigned_short_values + offset, n) > + != ref_sum_u16_u32 (unsigned_short_values + offset, n)) > + || (sum_s32_s64 (signed_int_values + offset, n) > + != ref_sum_s32_s64 (signed_int_values + offset, n)) > + || (sum_u32_u64 (unsigned_int_values + offset, n) > + != ref_sum_u32_u64 (unsigned_int_values + offset, n)) > + || (sum_abs_s32_u64 (signed_int_values + offset, n) > + != ref_abs_s32_u64 (signed_int_values + offset, n))) > + abort (); > + } > + return 0; > +} > -- > 2.50.1 (Apple Git-155)
