aarch64_<sra_op>rshr_n<mode><vczle><vczbe>_insn and
aarch64_<sra_op>rsra_n<mode>_insn model the rounding add and the shift in
<V2XWIDE>, as SRSHR/URSHR and SRSRA/URSRA do, but take the shift amount
as a vector in <MODE>. Only their own expanders build that, so nothing
but the RTL they emit themselves can ever match them: they are reachable
from the ACLE and from nothing else. The same operation written in C
reaches combine with the shift amount in the mode of the shift, so it
fails to match and comes out as movi + uaddw + shrn.
Take the shift amount in <V2XWIDE> in both patterns and build it there in
both expanders. The accepted range is unchanged; it comes from
aarch64_simd_shift_imm_<vec_or_offset>_<Vel>, which is keyed on <MODE>.
The two have to move together, because combine turns an add of a rounding
shift into SRSRA and folds SRSRA against a zero accumulator back into
SRSHR, so converting one alone would cost that fusion.
The widened shape is the only correct one: the source-width spelling
(x + (1 << (n - 1))) >> n is a different operation and is deliberately not
matched, since under qemu vrshrq_n_s32 (INT_MAX, 8) is 8388608 where the
C expression is -8388608. Only the 64-bit-result forms become reachable
from C; srshr v.16b, v.8h and v.4s stay intrinsic-only because their
<V2XWIDE> modes are V16HI, V8SI and V4DI, which the vectoriser does not
use. The RSHRN, RSHRN2 and SQRSHRUN recognisers keep the same
construction and stay ACLE-only for the same reason; 1/2 gives the RSHRN
pair source-width companions instead, which the saturating forms cannot
have.
For an eight-byte uint8_t block at -O2, movi + ldr + uaddw + shrn + str
becomes ldr + urshr + str.
Cross-tested aarch64-linux-gnu under qemu-aarch64 (aarch64.exp, simd.exp,
advsimd-intrinsics.exp, vect.exp, like for like against the same tree
unpatched) and bootstrapped on x86_64-pc-linux-gnu.
Assisted-by: Claude Opus 5 (Anthropic)
gcc/ChangeLog:
* config/aarch64/aarch64-simd.md
(aarch64_<sra_op>rsra_n<mode>_insn): Take the shift amount in
<V2XWIDE> mode.
(aarch64_<sra_op>rsra_n<mode>): Build the shift amount vector in
that mode.
(aarch64_<sra_op>rshr_n<mode><vczle><vczbe>_insn): Take the shift
amount in <V2XWIDE> mode.
(aarch64_<sra_op>rshr_n<mode>): Build the shift amount vector in
that mode.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/simd_srshr_1.c: New test.
* gcc.target/aarch64/simd_srshr_2.c: New test.
Signed-off-by: Dominic P <[email protected]>
---
gcc/config/aarch64/aarch64-simd.md | 10 ++-
.../gcc.target/aarch64/simd_srshr_1.c | 90 +++++++++++++++++++
.../gcc.target/aarch64/simd_srshr_2.c | 62 +++++++++++++
3 files changed, 158 insertions(+), 4 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/simd_srshr_1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/simd_srshr_2.c
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index a07e43d91d3..451ea9ff1a5 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -1635,7 +1635,8 @@
(<SHIFTEXTEND>:<V2XWIDE>
(match_operand:VSDQ_I_DI 2 "register_operand" "w"))
(match_operand:<V2XWIDE> 4 "aarch64_int_rnd_operand"))
- (match_operand:VSDQ_I_DI 3
"aarch64_simd_shift_imm_<vec_or_offset>_<Vel>")))
+ (match_operand:<V2XWIDE> 3
+ "aarch64_simd_shift_imm_<vec_or_offset>_<Vel>")))
(match_operand:VSDQ_I_DI 1 "register_operand" "0")))]
"TARGET_SIMD
&& aarch64_const_vec_rnd_cst_p (operands[4], operands[3])"
@@ -1674,7 +1675,7 @@
rtx rnd = immed_wide_int_const (rnd_wi, GET_MODE_INNER (<V2XWIDE>mode));
if (VECTOR_MODE_P (<MODE>mode))
{
- shft = gen_const_vec_duplicate (<MODE>mode, shft);
+ shft = gen_const_vec_duplicate (<V2XWIDE>mode, shft);
rnd = gen_const_vec_duplicate (<V2XWIDE>mode, rnd);
}
@@ -7393,7 +7394,8 @@
(<SHIFTEXTEND>:<V2XWIDE>
(match_operand:VSDQ_I_DI 1 "register_operand" "w"))
(match_operand:<V2XWIDE> 3 "aarch64_int_rnd_operand"))
- (match_operand:VSDQ_I_DI 2
"aarch64_simd_shift_imm_<vec_or_offset>_<Vel>"))))]
+ (match_operand:<V2XWIDE> 2
+ "aarch64_simd_shift_imm_<vec_or_offset>_<Vel>"))))]
"TARGET_SIMD
&& aarch64_const_vec_rnd_cst_p (operands[3], operands[2])"
"<sra_op>rshr\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
@@ -7416,7 +7418,7 @@
rtx rnd = immed_wide_int_const (rnd_wi, GET_MODE_INNER (<V2XWIDE>mode));
if (VECTOR_MODE_P (<MODE>mode))
{
- shft = gen_const_vec_duplicate (<MODE>mode, shft);
+ shft = gen_const_vec_duplicate (<V2XWIDE>mode, shft);
rnd = gen_const_vec_duplicate (<V2XWIDE>mode, rnd);
}
diff --git a/gcc/testsuite/gcc.target/aarch64/simd_srshr_1.c
b/gcc/testsuite/gcc.target/aarch64/simd_srshr_1.c
new file mode 100644
index 00000000000..aec88349ccd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd_srshr_1.c
@@ -0,0 +1,90 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include <stdint.h>
+
+typedef int8_t v8qi __attribute__((vector_size (8)));
+typedef int16_t v8hi __attribute__((vector_size (16)));
+typedef uint8_t v8uqi __attribute__((vector_size (8)));
+typedef uint16_t v8uhi __attribute__((vector_size (16)));
+typedef int16_t v4hi __attribute__((vector_size (8)));
+typedef int32_t v4si __attribute__((vector_size (16)));
+typedef uint16_t v4uhi __attribute__((vector_size (8)));
+typedef uint32_t v4usi __attribute__((vector_size (16)));
+
+/*
+** s8:
+** srshr v0\.8b, v0\.8b, 3
+** ret
+*/
+v8qi
+s8 (v8qi x)
+{
+ v8hi w = __builtin_convertvector (x, v8hi);
+ return __builtin_convertvector ((w + 4) >> 3, v8qi);
+}
+
+/*
+** u8:
+** urshr v0\.8b, v0\.8b, 3
+** ret
+*/
+v8uqi
+u8 (v8uqi x)
+{
+ v8uhi w = __builtin_convertvector (x, v8uhi);
+ return __builtin_convertvector ((w + 4) >> 3, v8uqi);
+}
+
+/*
+** s16:
+** srshr v0\.4h, v0\.4h, 6
+** ret
+*/
+v4hi
+s16 (v4hi x)
+{
+ v4si w = __builtin_convertvector (x, v4si);
+ return __builtin_convertvector ((w + 32) >> 6, v4hi);
+}
+
+/*
+** u16:
+** urshr v0\.4h, v0\.4h, 6
+** ret
+*/
+v4uhi
+u16 (v4uhi x)
+{
+ v4usi w = __builtin_convertvector (x, v4usi);
+ return __builtin_convertvector ((w + 32) >> 6, v4uhi);
+}
+
+/*
+** loop_s8:
+** ldr d31, \[x1\]
+** srshr v31\.8b, v31\.8b, 3
+** str d31, \[x0\]
+** ret
+*/
+void
+loop_s8 (int8_t *__restrict d, const int8_t *__restrict s)
+{
+ for (int i = 0; i < 8; i++)
+ d[i] = (int8_t) ((s[i] + 4) >> 3);
+}
+
+/*
+** loop_u8:
+** ldr d31, \[x1\]
+** urshr v31\.8b, v31\.8b, 3
+** str d31, \[x0\]
+** ret
+*/
+void
+loop_u8 (uint8_t *__restrict d, const uint8_t *__restrict s)
+{
+ for (int i = 0; i < 8; i++)
+ d[i] = (uint8_t) ((s[i] + 4) >> 3);
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/simd_srshr_2.c
b/gcc/testsuite/gcc.target/aarch64/simd_srshr_2.c
new file mode 100644
index 00000000000..51129fdd165
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd_srshr_2.c
@@ -0,0 +1,62 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -march=armv8-a" } */
+
+#include <stdint.h>
+
+typedef int8_t v8qi __attribute__((vector_size (8)));
+typedef int16_t v8hi __attribute__((vector_size (16)));
+typedef uint8_t v8uqi __attribute__((vector_size (8)));
+typedef uint16_t v8uhi __attribute__((vector_size (16)));
+typedef int16_t v4hi __attribute__((vector_size (8)));
+typedef int32_t v4si __attribute__((vector_size (16)));
+typedef uint16_t v4uhi __attribute__((vector_size (8)));
+typedef uint32_t v4usi __attribute__((vector_size (16)));
+
+#define ROUND(NAME, NARROW, WIDE, ELT, WELT, LANES, N) \
+ __attribute__((noipa)) static NARROW \
+ NAME (NARROW x) \
+ { \
+ WIDE w = __builtin_convertvector (x, WIDE);
\
+ return __builtin_convertvector ((w + (1 << (N - 1))) >> N, NARROW);
\
+ } \
+ __attribute__((noipa)) static void \
+ NAME##_check (void) \
+ { \
+ NARROW x, r; \
+ for (int rep = 0; rep < 4096; rep++) \
+ {
\
+ for (int i = 0; i < LANES; i++) \
+ x[i] = (ELT) rnd (); \
+ r = NAME (x); \
+ for (int i = 0; i < LANES; i++) \
+ { \
+ WELT w = (WELT) x[i] + (1 << (N - 1)); \
+ if (r[i] != (ELT) (w >> N)) \
+ __builtin_abort (); \
+ } \
+ }
\
+ }
+
+static uint64_t seed = 0x123456789abcdefULL;
+
+static uint64_t
+rnd (void)
+{
+ seed = seed * 6364136223846793005ULL + 1442695040888963407ULL;
+ return seed >> 11;
+}
+
+ROUND (s8, v8qi, v8hi, int8_t, int16_t, 8, 3)
+ROUND (u8, v8uqi, v8uhi, uint8_t, uint16_t, 8, 3)
+ROUND (s16, v4hi, v4si, int16_t, int32_t, 4, 6)
+ROUND (u16, v4uhi, v4usi, uint16_t, uint32_t, 4, 6)
+
+int
+main (void)
+{
+ s8_check ();
+ u8_check ();
+ s16_check ();
+ u16_check ();
+ return 0;
+}
--
2.55.0
