From: Kyrylo Tkachov <[email protected]>
The 64-bit accumulator path in precompute_partition_info_sums_ from
Xiph FLAC adds the absolute values of 32-bit residuals.
The relevant statement in src/libFLAC/stream_encoder.c is:
abs_residual_partition_sum64 += abs (residual[residual_sample]);
The vectorizer recognizes this as a widening sum. The SVE2
reduc_widen_ssum expander then emits SADDWB followed by SADDWT. Both
instructions add one half of the narrow vector to the same accumulator.
SADALP performs the same pairwise widening accumulation in one
instruction. The unsigned widening sum has the same issue with UADDWB
and UADDWT.
Emit the existing predicated SADALP and UADALP patterns from the SVE2
widening-sum expanders. Use an all-true predicate because the optab is
unpredicated. Keep the existing SVE2.1 and SME2 H-to-S dot-product case.
With:
-O3 -mcpu=neoverse-v2 -mautovec-preference=sve-only
-msve-vector-bits=scalable
--param aarch64-vect-unroll-limit=1
--param vect-partial-vector-usage=0
the steady loop for the FLAC reduction changes from:
.L4:
ld1w z30.s, p7/z, [x0, x2, lsl 2]
mov x4, x2
add x2, x2, x5
abs z30.s, p7/m, z30.s
saddwb z31.d, z31.d, z30.s
saddwt z31.d, z31.d, z30.s
cmp w3, w2
bcs .L4
to:
.L4:
ld1w z30.s, p7/z, [x0, x2, lsl 2]
mov x4, x2
add x2, x2, x5
abs z30.s, p7/m, z30.s
sadalp z31.d, p7/m, z30.s
cmp w3, w2
bcs .L4
Bootstrapped and tested on aarch64-none-linux-gnu.
Ok for trunk?
Thanks,
Kyrill
Cc: Tamar Christina <[email protected]>
gcc/ChangeLog:
* config/aarch64/aarch64-sve2.md
(reduc_widen_ssum<mode><Vnarrow>3): Emit cond_sadalp.
(reduc_widen_usum<mode><Vnarrow>3): Emit cond_uadalp.
(cond_<sve_int_op><mode>, *cond_<sve_int_op><mode>_2,
*cond_<sve_int_op><mode>_z): Fix comments.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve2/pr122069_1.c: Expect UADALP.
* gcc.target/aarch64/sve2/widen-sum-adalp-1.c: New test.
* gcc.target/aarch64/sve2/widen-sum-adalp-run.c: Likewise.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/config/aarch64/aarch64-sve2.md | 47 +++----
.../gcc.target/aarch64/sve2/pr122069_1.c | 3 +-
.../aarch64/sve2/widen-sum-adalp-1.c | 38 ++++++
.../aarch64/sve2/widen-sum-adalp-run.c | 123 ++++++++++++++++++
4 files changed, 183 insertions(+), 28 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c
diff --git a/gcc/config/aarch64/aarch64-sve2.md
b/gcc/config/aarch64/aarch64-sve2.md
index c33df283cc2..7d8504db304 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -2563,18 +2563,11 @@
[(set_attr "sve_type" "sve_int_general")]
)
-;; Define single step widening for reduc_widen_ssum using SADDWB and SADDWT
+;; Define single step widening for reduc_widen_ssum using SADALP.
(define_expand "reduc_widen_ssum<mode><Vnarrow>3"
- [(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
- (unspec:SVE_FULL_HSDI
- [(match_operand:SVE_FULL_HSDI 2 "register_operand")
- (match_operand:<VNARROW> 1 "register_operand")]
- UNSPEC_SADDWB))
- (set (match_dup 0)
- (unspec:SVE_FULL_HSDI
- [(match_dup 0)
- (match_dup 1)]
- UNSPEC_SADDWT))]
+ [(match_operand:SVE_FULL_HSDI 0 "register_operand")
+ (match_operand:<VNARROW> 1 "register_operand")
+ (match_operand:SVE_FULL_HSDI 2 "register_operand")]
"TARGET_SVE2"
{
/* Use dot product to perform double widening sum reductions by
@@ -2588,20 +2581,18 @@
ones, operands[2]));
DONE;
}
+
+ rtx pred = aarch64_ptrue_reg (<VPRED>mode);
+ emit_insn (gen_cond_sadalp<mode> (operands[0], pred, operands[2],
+ operands[1], operands[2]));
+ DONE;
})
-;; Define single step widening for reduc_widen_usum using UADDWB and UADDWT
+;; Define single step widening for reduc_widen_usum using UADALP.
(define_expand "reduc_widen_usum<mode><Vnarrow>3"
- [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w")
- (unspec:SVE_FULL_HSDI
- [(match_operand:SVE_FULL_HSDI 2 "register_operand" "w")
- (match_operand:<VNARROW> 1 "register_operand" "w")]
- UNSPEC_UADDWB))
- (set (match_dup 0)
- (unspec:SVE_FULL_HSDI
- [(match_dup 0)
- (match_dup 1)]
- UNSPEC_UADDWT))]
+ [(match_operand:SVE_FULL_HSDI 0 "register_operand")
+ (match_operand:<VNARROW> 1 "register_operand")
+ (match_operand:SVE_FULL_HSDI 2 "register_operand")]
"TARGET_SVE2"
{
if (TARGET_SVE2p1_OR_SME2
@@ -2613,6 +2604,11 @@
ones, operands[2]));
DONE;
}
+
+ rtx pred = aarch64_ptrue_reg (<VPRED>mode);
+ emit_insn (gen_cond_uadalp<mode> (operands[0], pred, operands[2],
+ operands[1], operands[2]));
+ DONE;
})
;; -------------------------------------------------------------------------
@@ -3344,7 +3340,7 @@
;; - UADALP
;; -------------------------------------------------------------------------
-;; Predicated pairwise absolute difference and accumulate with merging.
+;; Predicated add long pairwise and accumulate with merging.
(define_expand "@cond_<sve_int_op><mode>"
[(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
(unspec:SVE_FULL_HSDI
@@ -3364,8 +3360,7 @@
|| CONSTANT_P (operands[4]));
})
-;; Predicated pairwise absolute difference and accumulate, merging with
-;; the first input.
+;; Predicated add long pairwise and accumulate, merging with the first input.
(define_insn_and_rewrite "*cond_<sve_int_op><mode>_2"
[(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
(unspec:SVE_FULL_HSDI
@@ -3389,7 +3384,7 @@
[(set_attr "sve_type" "sve_int_accum")]
)
-;; Predicated pairwise absolute difference and accumulate, merging with zero.
+;; Predicated add long pairwise and accumulate, merging with zero.
(define_insn_and_rewrite "*cond_<sve_int_op><mode>_z"
[(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
(unspec:SVE_FULL_HSDI
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c
b/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c
index 6a347072ae8..02e38f70b54 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pr122069_1.c
@@ -24,8 +24,7 @@ int foo_int(unsigned char *x, unsigned char * restrict y) {
** foo2_int:
** ...
** add z[0-9]+.h, z[0-9]+.h, z[0-9]+.h
-** uaddwb z[0-9]+.s, z[0-9]+.s, z[0-9]+.h
-** uaddwt z[0-9]+.s, z[0-9]+.s, z[0-9]+.h
+** uadalp z[0-9]+.s, p[0-7]/m, z[0-9]+.h
** ...
*/
int foo2_int(unsigned short *x, unsigned short * restrict y) {
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c
b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c
new file mode 100644
index 00000000000..172d6ba645f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv9-a+sve2 -mautovec-preference=sve-only
-msve-vector-bits=scalable --param aarch64-vect-unroll-limit=1 --param
vect-partial-vector-usage=0" } */
+
+#define DEF(NAME, INPUT, OUTPUT) \
+ OUTPUT \
+ NAME (const INPUT *a, unsigned int n) \
+ { \
+ OUTPUT sum = 0; \
+ for (unsigned int i = 0; i < n; ++i) \
+ sum += a[i]; \
+ return sum; \
+ }
+
+DEF (sum_s8_s16, signed char, short)
+DEF (sum_u8_u16, unsigned char, unsigned short)
+DEF (sum_s16_s32, short, int)
+DEF (sum_u16_u32, unsigned short, unsigned int)
+DEF (sum_s32_s64, int, long long)
+DEF (sum_u32_u64, unsigned int, unsigned long long)
+
+unsigned long long
+sum_abs_s32_u64 (const int *a, unsigned int n)
+{
+ unsigned long long sum = 0;
+ for (unsigned int i = 0; i < n; ++i)
+ sum += __builtin_abs (a[i]);
+ return sum;
+}
+
+/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.h, p[0-7]/m,
z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.h, p[0-7]/m,
z[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.s, p[0-7]/m,
z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.s, p[0-7]/m,
z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tsadalp\tz[0-9]+\.d, p[0-7]/m,
z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tuadalp\tz[0-9]+\.d, p[0-7]/m,
z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */
+/* { dg-final { scan-assembler-not {\tsaddw[bt]\t} } } */
+/* { dg-final { scan-assembler-not {\tuaddw[bt]\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c
b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c
new file mode 100644
index 00000000000..d064ed99d0c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/widen-sum-adalp-run.c
@@ -0,0 +1,123 @@
+/* { dg-do run { target aarch64_sve2_hw } } */
+/* { dg-options "-O3 -march=armv9-a+sve2 -mautovec-preference=sve-only
-msve-vector-bits=scalable --param aarch64-vect-unroll-limit=1 --param
vect-partial-vector-usage=0" } */
+
+#include <arm_sve.h>
+#include <limits.h>
+#include <stdlib.h>
+
+#define DEF_SUMS(NAME, INPUT, OUTPUT) \
+ __attribute__((noipa)) OUTPUT \
+ NAME (const INPUT *a, unsigned int n) \
+ { \
+ OUTPUT sum = 0; \
+ for (unsigned int i = 0; i < n; ++i) \
+ sum += a[i]; \
+ return sum; \
+ } \
+ \
+ __attribute__((noipa, optimize ("no-tree-vectorize"))) OUTPUT \
+ ref_##NAME (const INPUT *a, unsigned int n) \
+ { \
+ OUTPUT sum = 0; \
+ for (unsigned int i = 0; i < n; ++i) \
+ sum += a[i]; \
+ return sum; \
+ }
+
+DEF_SUMS (sum_s8_s16, signed char, short)
+DEF_SUMS (sum_u8_u16, unsigned char, unsigned short)
+DEF_SUMS (sum_s16_s32, short, int)
+DEF_SUMS (sum_u16_u32, unsigned short, unsigned int)
+DEF_SUMS (sum_s32_s64, int, long long)
+DEF_SUMS (sum_u32_u64, unsigned int, unsigned long long)
+
+__attribute__((noipa)) unsigned long long
+sum_abs_s32_u64 (const int *a, unsigned int n)
+{
+ unsigned long long sum = 0;
+ for (unsigned int i = 0; i < n; ++i)
+ sum += __builtin_abs (a[i]);
+ return sum;
+}
+
+__attribute__((noipa, optimize ("no-tree-vectorize"))) unsigned long long
+ref_abs_s32_u64 (const int *a, unsigned int n)
+{
+ unsigned long long sum = 0;
+ for (unsigned int i = 0; i < n; ++i)
+ sum += __builtin_abs (a[i]);
+ return sum;
+}
+
+int
+main (void)
+{
+ unsigned int vl_b = svcntb ();
+ unsigned int vl_h = svcnth ();
+ unsigned int vl_w = svcntw ();
+ unsigned int count = 4 * vl_b + 3;
+ unsigned int allocation_count = count + 1;
+ signed char *signed_char_values
+ = (signed char *) __builtin_alloca (allocation_count
+ * sizeof (signed char));
+ unsigned char *unsigned_char_values
+ = (unsigned char *) __builtin_alloca (allocation_count
+ * sizeof (unsigned char));
+ short *signed_short_values
+ = (short *) __builtin_alloca (allocation_count * sizeof (short));
+ unsigned short *unsigned_short_values
+ = (unsigned short *) __builtin_alloca (allocation_count
+ * sizeof (unsigned short));
+ int *signed_int_values
+ = (int *) __builtin_alloca (allocation_count * sizeof (int));
+ unsigned int *unsigned_int_values
+ = (unsigned int *) __builtin_alloca (allocation_count
+ * sizeof (unsigned int));
+
+ for (unsigned int i = 0; i < allocation_count; ++i)
+ {
+ static const signed char signed_char_edges[] =
+ { SCHAR_MIN, SCHAR_MAX, 1, 0, -1 };
+ static const unsigned char unsigned_char_edges[] =
+ { 0, 1, UCHAR_MAX, UCHAR_MAX - 1, 0x80 };
+ static const short signed_short_edges[] =
+ { SHRT_MIN, SHRT_MAX, 1, 0, -1 };
+ static const unsigned short unsigned_short_edges[] =
+ { 0, 1, USHRT_MAX, USHRT_MAX - 1, 0x8000 };
+ static const int signed_int_edges[] =
+ { INT_MIN + 1, INT_MAX, -1, 0, 1, -123456789, 123456789 };
+ static const unsigned int unsigned_int_edges[] =
+ { 0, 1, UINT_MAX, UINT_MAX - 1, 0x80000000U, 123456789U };
+ signed_char_values[i] = signed_char_edges[i % 5];
+ unsigned_char_values[i] = unsigned_char_edges[i % 5];
+ signed_short_values[i] = signed_short_edges[i % 5];
+ unsigned_short_values[i] = unsigned_short_edges[i % 5];
+ signed_int_values[i] = signed_int_edges[i % 7];
+ unsigned_int_values[i] = unsigned_int_edges[i % 6];
+ }
+
+ unsigned int lengths[] =
+ { 0, 1, vl_w - 1, vl_w, vl_w + 1, vl_h - 1, vl_h, vl_h + 1,
+ vl_b - 1, vl_b, vl_b + 1, 2 * vl_b + 1, count };
+ for (unsigned int i = 0; i < sizeof (lengths) / sizeof (lengths[0]); ++i)
+ for (unsigned int offset = 0; offset <= 1; ++offset)
+ {
+ unsigned int n = lengths[i];
+ if ((sum_s8_s16 (signed_char_values + offset, n)
+ != ref_sum_s8_s16 (signed_char_values + offset, n))
+ || (sum_u8_u16 (unsigned_char_values + offset, n)
+ != ref_sum_u8_u16 (unsigned_char_values + offset, n))
+ || (sum_s16_s32 (signed_short_values + offset, n)
+ != ref_sum_s16_s32 (signed_short_values + offset, n))
+ || (sum_u16_u32 (unsigned_short_values + offset, n)
+ != ref_sum_u16_u32 (unsigned_short_values + offset, n))
+ || (sum_s32_s64 (signed_int_values + offset, n)
+ != ref_sum_s32_s64 (signed_int_values + offset, n))
+ || (sum_u32_u64 (unsigned_int_values + offset, n)
+ != ref_sum_u32_u64 (unsigned_int_values + offset, n))
+ || (sum_abs_s32_u64 (signed_int_values + offset, n)
+ != ref_abs_s32_u64 (signed_int_values + offset, n)))
+ abort ();
+ }
+ return 0;
+}
--
2.50.1 (Apple Git-155)
