https://gcc.gnu.org/g:c7e3f9e456829642e72fac4ffc3f7ce08d16a526
commit r17-5327-gc7e3f9e456829642e72fac4ffc3f7ce08d16a526 Author: Kyrylo Tkachov <[email protected]> Date: Thu Aug 27 13:26:09 2026 +0200 aarch64: Use SVE2 ADALP for widening sums The 64-bit accumulator path in precompute_partition_info_sums_ from Xiph FLAC adds the absolute values of 32-bit residuals. The relevant statement in src/libFLAC/stream_encoder.c is: abs_residual_partition_sum64 += abs (residual[residual_sample]); The vectorizer recognizes this as a widening sum. The SVE2 reduc_widen_ssum expander then emits SADDWB followed by SADDWT. Both instructions add one half of the narrow vector to the same accumulator. SADALP performs the same pairwise widening accumulation in one instruction. The unsigned widening sum has the same issue with UADDWB and UADDWT. Emit the existing predicated SADALP and UADALP patterns from the SVE2 widening-sum expanders. Use an all-true predicate because the optab is unpredicated. Keep the existing SVE2.1 and SME2 H-to-S dot-product case. With: -O3 -mcpu=neoverse-v2 -mautovec-preference=sve-only -msve-vector-bits=scalable --param aarch64-vect-unroll-limit=1 --param vect-partial-vector-usage=0 the steady loop for the FLAC reduction changes from: .L4: ld1w z30.s, p7/z, [x0, x2, lsl 2] mov x4, x2 add x2, x2, x5 abs z30.s, p7/m, z30.s saddwb z31.d, z31.d, z30.s saddwt z31.d, z31.d, z30.s cmp w3, w2 bcs .L4 to: .L4: ld1w z30.s, p7/z, [x0, x2, lsl 2] mov x4, x2 add x2, x2, x5 abs z30.s, p7/m, z30.s sadalp z31.d, p7/m, z30.s cmp w3, w2 bcs .L4 Bootstrapped and tested on aarch64-none-linux-gnu. gcc/ChangeLog: * config/aarch64/aarch64-sve2.md (reduc_widen_ssum<mode><Vnarrow>3): Emit cond_sadalp. (reduc_widen_usum<mode><Vnarrow>3): Emit cond_uadalp. (cond_<sve_int_op><mode>, *cond_<sve_int_op><mode>_2, *cond_<sve_int_op><mode>_z): Fix comments. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sve2/pr122069_1.c: Expect UADALP. * gcc.target/aarch64/sve2/widen-sum-adalp-1.c: New test. * gcc.target/aarch64/sve2/widen-sum-adalp-run.c: Likewise. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/config/aarch64/aarch64-sve2.md | 47 ++++---- gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c | 3 +- .../gcc.target/aarch64/sve2/widen-sum-adalp-1.c | 38 +++++++ .../gcc.target/aarch64/sve2/widen-sum-adalp-run.c | 123 +++++++++++++++++++++ 4 files changed, 183 insertions(+), 28 deletions(-) diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index c33df283cc2a..7d8504db3048 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -2563,18 +2563,11 @@ [(set_attr "sve_type" "sve_int_general")] ) -;; Define single step widening for reduc_widen_ssum using SADDWB and SADDWT +;; Define single step widening for reduc_widen_ssum using SADALP. (define_expand "reduc_widen_ssum<mode><Vnarrow>3" - [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") - (unspec:SVE_FULL_HSDI - [(match_operand:SVE_FULL_HSDI 2 "register_operand") - (match_operand:<VNARROW> 1 "register_operand")] - UNSPEC_SADDWB)) - (set (match_dup 0) - (unspec:SVE_FULL_HSDI - [(match_dup 0) - (match_dup 1)] - UNSPEC_SADDWT))] + [(match_operand:SVE_FULL_HSDI 0 "register_operand") + (match_operand:<VNARROW> 1 "register_operand") + (match_operand:SVE_FULL_HSDI 2 "register_operand")] "TARGET_SVE2" { /* Use dot product to perform double widening sum reductions by @@ -2588,20 +2581,18 @@ ones, operands[2])); DONE; } + + rtx pred = aarch64_ptrue_reg (<VPRED>mode); + emit_insn (gen_cond_sadalp<mode> (operands[0], pred, operands[2], + operands[1], operands[2])); + DONE; }) -;; Define single step widening for reduc_widen_usum using UADDWB and UADDWT +;; Define single step widening for reduc_widen_usum using UADALP. (define_expand "reduc_widen_usum<mode><Vnarrow>3" - [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w") - (unspec:SVE_FULL_HSDI - [(match_operand:SVE_FULL_HSDI 2 "register_operand" "w") - (match_operand:<VNARROW> 1 "register_operand" "w")] - UNSPEC_UADDWB)) - (set (match_dup 0) - (unspec:SVE_FULL_HSDI - [(match_dup 0) - (match_dup 1)] - UNSPEC_UADDWT))] + [(match_operand:SVE_FULL_HSDI 0 "register_operand") + (match_operand:<VNARROW> 1 "register_operand") + (match_operand:SVE_FULL_HSDI 2 "register_operand")] "TARGET_SVE2" { if (TARGET_SVE2p1_OR_SME2 @@ -2613,6 +2604,11 @@ ones, operands[2])); DONE; } + + rtx pred = aarch64_ptrue_reg (<VPRED>mode); + emit_insn (gen_cond_uadalp<mode> (operands[0], pred, operands[2], + operands[1], operands[2])); + DONE; }) ;; ------------------------------------------------------------------------- @@ -3344,7 +3340,7 @@ ;; - UADALP ;; ------------------------------------------------------------------------- -;; Predicated pairwise absolute difference and accumulate with merging. +;; Predicated add long pairwise and accumulate with merging. (define_expand "@cond_<sve_int_op><mode>" [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") (unspec:SVE_FULL_HSDI @@ -3364,8 +3360,7 @@ || CONSTANT_P (operands[4])); }) -;; Predicated pairwise absolute difference and accumulate, merging with -;; the first input. +;; Predicated add long pairwise and accumulate, merging with the first input. (define_insn_and_rewrite "*cond_<sve_int_op><mode>_2" [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") (unspec:SVE_FULL_HSDI @@ -3389,7 +3384,7 @@ [(set_attr "sve_type" "sve_int_accum")] ) -;; Predicated pairwise absolute difference and accumulate, merging with zero. +;; Predicated add long pairwise and accumulate, merging with zero. (define_insn_and_rewrite "*cond_<sve_int_op><mode>_z" [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") (unspec:SVE_FULL_HSDI diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c index 6a347072ae89..02e38f70b54d 100644 --- a/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c +++ b/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c @@ -24,8 +24,7 @@ int foo_int(unsigned char *x, unsigned char * restrict y) { ** foo2_int: ** ... ** add z[0-9]+.h, z[0-9]+.h, z[0-9]+.h -** uaddwb z[0-9]+.s, z[0-9]+.s, z[0-9]+.h -** uaddwt z[0-9]+.s, z[0-9]+.s, z[0-9]+.h +** uadalp z[0-9]+.s, p[0-7]/m, z[0-9]+.h ** ... */ int foo2_int(unsigned short *x, unsigned short * restrict y) { diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c new file mode 100644 index 000000000000..172d6ba645f0 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c @@ -0,0 +1,38 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv9-a+sve2 -mautovec-preference=sve-only -msve-vector-bits=scalable --param aarch64-vect-unroll-limit=1 --param vect-partial-vector-usage=0" } */ + +#define DEF(NAME, INPUT, OUTPUT) \ + OUTPUT \ + NAME (const INPUT *a, unsigned int n) \ + { \ + OUTPUT sum = 0; \ + for (unsigned int i = 0; i < n; ++i) \ + sum += a[i]; \ + return sum; \ + } + +DEF (sum_s8_s16, signed char, short) +DEF (sum_u8_u16, unsigned char, unsigned short) +DEF (sum_s16_s32, short, int) +DEF (sum_u16_u32, unsigned short, unsigned int) +DEF (sum_s32_s64, int, long long) +DEF (sum_u32_u64, unsigned int, unsigned long long) + +unsigned long long +sum_abs_s32_u64 (const int *a, unsigned int n) +{ + unsigned long long sum = 0; + for (unsigned int i = 0; i < n; ++i) + sum += __builtin_abs (a[i]); + return sum; +} + +/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.b\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.b\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */ +/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */ +/* { dg-final { scan-assembler-not {\tsaddw[bt]\t} } } */ +/* { dg-final { scan-assembler-not {\tuaddw[bt]\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c new file mode 100644 index 000000000000..d064ed99d0c7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c @@ -0,0 +1,123 @@ +/* { dg-do run { target aarch64_sve2_hw } } */ +/* { dg-options "-O3 -march=armv9-a+sve2 -mautovec-preference=sve-only -msve-vector-bits=scalable --param aarch64-vect-unroll-limit=1 --param vect-partial-vector-usage=0" } */ + +#include <arm_sve.h> +#include <limits.h> +#include <stdlib.h> + +#define DEF_SUMS(NAME, INPUT, OUTPUT) \ + __attribute__((noipa)) OUTPUT \ + NAME (const INPUT *a, unsigned int n) \ + { \ + OUTPUT sum = 0; \ + for (unsigned int i = 0; i < n; ++i) \ + sum += a[i]; \ + return sum; \ + } \ + \ + __attribute__((noipa, optimize ("no-tree-vectorize"))) OUTPUT \ + ref_##NAME (const INPUT *a, unsigned int n) \ + { \ + OUTPUT sum = 0; \ + for (unsigned int i = 0; i < n; ++i) \ + sum += a[i]; \ + return sum; \ + } + +DEF_SUMS (sum_s8_s16, signed char, short) +DEF_SUMS (sum_u8_u16, unsigned char, unsigned short) +DEF_SUMS (sum_s16_s32, short, int) +DEF_SUMS (sum_u16_u32, unsigned short, unsigned int) +DEF_SUMS (sum_s32_s64, int, long long) +DEF_SUMS (sum_u32_u64, unsigned int, unsigned long long) + +__attribute__((noipa)) unsigned long long +sum_abs_s32_u64 (const int *a, unsigned int n) +{ + unsigned long long sum = 0; + for (unsigned int i = 0; i < n; ++i) + sum += __builtin_abs (a[i]); + return sum; +} + +__attribute__((noipa, optimize ("no-tree-vectorize"))) unsigned long long +ref_abs_s32_u64 (const int *a, unsigned int n) +{ + unsigned long long sum = 0; + for (unsigned int i = 0; i < n; ++i) + sum += __builtin_abs (a[i]); + return sum; +} + +int +main (void) +{ + unsigned int vl_b = svcntb (); + unsigned int vl_h = svcnth (); + unsigned int vl_w = svcntw (); + unsigned int count = 4 * vl_b + 3; + unsigned int allocation_count = count + 1; + signed char *signed_char_values + = (signed char *) __builtin_alloca (allocation_count + * sizeof (signed char)); + unsigned char *unsigned_char_values + = (unsigned char *) __builtin_alloca (allocation_count + * sizeof (unsigned char)); + short *signed_short_values + = (short *) __builtin_alloca (allocation_count * sizeof (short)); + unsigned short *unsigned_short_values + = (unsigned short *) __builtin_alloca (allocation_count + * sizeof (unsigned short)); + int *signed_int_values + = (int *) __builtin_alloca (allocation_count * sizeof (int)); + unsigned int *unsigned_int_values + = (unsigned int *) __builtin_alloca (allocation_count + * sizeof (unsigned int)); + + for (unsigned int i = 0; i < allocation_count; ++i) + { + static const signed char signed_char_edges[] = + { SCHAR_MIN, SCHAR_MAX, 1, 0, -1 }; + static const unsigned char unsigned_char_edges[] = + { 0, 1, UCHAR_MAX, UCHAR_MAX - 1, 0x80 }; + static const short signed_short_edges[] = + { SHRT_MIN, SHRT_MAX, 1, 0, -1 }; + static const unsigned short unsigned_short_edges[] = + { 0, 1, USHRT_MAX, USHRT_MAX - 1, 0x8000 }; + static const int signed_int_edges[] = + { INT_MIN + 1, INT_MAX, -1, 0, 1, -123456789, 123456789 }; + static const unsigned int unsigned_int_edges[] = + { 0, 1, UINT_MAX, UINT_MAX - 1, 0x80000000U, 123456789U }; + signed_char_values[i] = signed_char_edges[i % 5]; + unsigned_char_values[i] = unsigned_char_edges[i % 5]; + signed_short_values[i] = signed_short_edges[i % 5]; + unsigned_short_values[i] = unsigned_short_edges[i % 5]; + signed_int_values[i] = signed_int_edges[i % 7]; + unsigned_int_values[i] = unsigned_int_edges[i % 6]; + } + + unsigned int lengths[] = + { 0, 1, vl_w - 1, vl_w, vl_w + 1, vl_h - 1, vl_h, vl_h + 1, + vl_b - 1, vl_b, vl_b + 1, 2 * vl_b + 1, count }; + for (unsigned int i = 0; i < sizeof (lengths) / sizeof (lengths[0]); ++i) + for (unsigned int offset = 0; offset <= 1; ++offset) + { + unsigned int n = lengths[i]; + if ((sum_s8_s16 (signed_char_values + offset, n) + != ref_sum_s8_s16 (signed_char_values + offset, n)) + || (sum_u8_u16 (unsigned_char_values + offset, n) + != ref_sum_u8_u16 (unsigned_char_values + offset, n)) + || (sum_s16_s32 (signed_short_values + offset, n) + != ref_sum_s16_s32 (signed_short_values + offset, n)) + || (sum_u16_u32 (unsigned_short_values + offset, n) + != ref_sum_u16_u32 (unsigned_short_values + offset, n)) + || (sum_s32_s64 (signed_int_values + offset, n) + != ref_sum_s32_s64 (signed_int_values + offset, n)) + || (sum_u32_u64 (unsigned_int_values + offset, n) + != ref_sum_u32_u64 (unsigned_int_values + offset, n)) + || (sum_abs_s32_u64 (signed_int_values + offset, n) + != ref_abs_s32_u64 (signed_int_values + offset, n))) + abort (); + } + return 0; +}
