https://gcc.gnu.org/g:238a3c1042262cb5f4107fb3a90c679e00d9d0c8
commit r17-4672-g238a3c1042262cb5f4107fb3a90c679e00d9d0c8 Author: Dominic P <[email protected]> Date: Thu Sep 10 01:30:24 2026 +0100 aarch64: Match a rounding narrowing shift written at the source width RSHRN is only reachable from the ACLE intrinsics. The nearest recognisers, *aarch64_<shrn_op>rshrn_n<mode>_insn and its high-half forms, do the rounding add and the shift in <V2XWIDE> over an extended operand -- the shape the intrinsic expanders build -- so C that writes (x + (1 << (n - 1))) >> n and narrows the result, which does both at the source width, gets movi + add + shrn/shrn2 instead. Widening those recognisers would need V16HI, V8SI and V4DI arithmetic, which the vectoriser does not produce, so add source-width companions. With W the source element width and w = W/2 the destination width, both forms keep bits [n+w-1:n] of x + 2^(n-1) and differ only above bit W-1, so they agree for n <= W/2; aarch64_int_rnd_operand and aarch64_simd_shift_imm_vec_<vn_mode> each cap n there. The saturating members of the family (SQRSHRN, SQRSHRUN, UQRSHRN) read the full shifted value and are deliberately left alone: vqrshrun_n_s16 (32767, 6) is 255 where the source-width C form is 0. The intrinsics gain too. A shift equal to the destination element width expands to RADDHN against a shareable zero, and combine now folds that zero away into a single RSHRN. advsimd-intrinsics/shrn-1.c therefore loses its movi and drops from six instructions to five, and shrn-2.c, shrn-3.c and shrn-4.c drop from four to three; their raddhn expectations are updated here to the shorter rshrn sequence. Combine does not run at -O0, so the three that lacked it gain the -O0 skip shrn-1.c carries. x265 dct.cpp's partialButterfly output stage, reduced to for (int i = 0; i < 64; i++) d[i] = (int16_t) ((s[i] + 128) >> 8); goes from 46 to 38 instructions at -O3 -mcpu=cortex-a76; ffmpeg's h264 chroma stage, (uint8_t) ((s[i] + 32) >> 6) over uint16_t, from 88 to 85. Cross-tested aarch64-linux-gnu under qemu-aarch64 (aarch64.exp, simd.exp, advsimd-intrinsics.exp, vect.exp, like for like against the same tree unpatched) and bootstrapped on x86_64-pc-linux-gnu. Assisted-by: Claude Opus 5 (Anthropic) gcc/ChangeLog: * config/aarch64/aarch64-simd.md (*aarch64_<sra_op>rshrn_n<mode>_insn<vczle><vczbe>): New pattern. (*aarch64_<sra_op>rshrn2_n<mode>_insn_le): Likewise. (*aarch64_<sra_op>rshrn2_n<mode>_insn_be): Likewise. gcc/testsuite/ChangeLog: * gcc.target/aarch64/advsimd-intrinsics/shrn-1.c: Expect rshrn and rshrn2 in place of raddhn and raddhn2, and no movi. * gcc.target/aarch64/advsimd-intrinsics/shrn-2.c: Expect rshrn and rshrn2 in place of raddhn and raddhn2. Skip at -O0. * gcc.target/aarch64/advsimd-intrinsics/shrn-3.c: Likewise. * gcc.target/aarch64/advsimd-intrinsics/shrn-4.c: Likewise. * gcc.target/aarch64/simd_rshrn_1.c: New test. * gcc.target/aarch64/simd_rshrn_2.c: New test. Signed-off-by: Dominic P <[email protected]> Diff: --- gcc/config/aarch64/aarch64-simd.md | 46 +++++++++ .../gcc.target/aarch64/advsimd-intrinsics/shrn-1.c | 6 +- .../gcc.target/aarch64/advsimd-intrinsics/shrn-2.c | 5 +- .../gcc.target/aarch64/advsimd-intrinsics/shrn-3.c | 5 +- .../gcc.target/aarch64/advsimd-intrinsics/shrn-4.c | 5 +- gcc/testsuite/gcc.target/aarch64/simd_rshrn_1.c | 106 +++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/simd_rshrn_2.c | 64 +++++++++++++ 7 files changed, 228 insertions(+), 9 deletions(-) diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index 9d1a929576c7..517b74d8f1a8 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -7609,6 +7609,20 @@ } ) +(define_insn "*aarch64_<sra_op>rshrn_n<mode>_insn<vczle><vczbe>" + [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") + (truncate:<VNARROWQ> + (SHIFTRT:VQN + (plus:VQN + (match_operand:VQN 1 "register_operand" "w") + (match_operand:VQN 3 "aarch64_int_rnd_operand")) + (match_operand:VQN 2 "aarch64_simd_shift_imm_vec_<vn_mode>"))))] + "TARGET_SIMD + && aarch64_const_vec_rnd_cst_p (operands[3], operands[2])" + "rshrn\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2" + [(set_attr "type" "neon_shift_imm_narrow_q")] +) + (define_insn "*aarch64_sqshrun_n<mode>_insn<vczle><vczbe>" [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") (us_truncate:<VNARROWQ> @@ -7822,6 +7836,38 @@ [(set_attr "type" "neon_shift_imm_narrow_q")] ) +(define_insn "*aarch64_<sra_op>rshrn2_n<mode>_insn_le" + [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w") + (vec_concat:<VNARROWQ2> + (match_operand:<VNARROWQ> 1 "register_operand" "0") + (truncate:<VNARROWQ> + (SHIFTRT:VQN + (plus:VQN + (match_operand:VQN 2 "register_operand" "w") + (match_operand:VQN 4 "aarch64_int_rnd_operand")) + (match_operand:VQN 3 "aarch64_simd_shift_imm_vec_<vn_mode>")))))] + "TARGET_SIMD && !BYTES_BIG_ENDIAN + && aarch64_const_vec_rnd_cst_p (operands[4], operands[3])" + "rshrn2\t%<vn2>0.<V2ntype>, %<v>2.<Vtype>, %3" + [(set_attr "type" "neon_shift_imm_narrow_q")] +) + +(define_insn "*aarch64_<sra_op>rshrn2_n<mode>_insn_be" + [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w") + (vec_concat:<VNARROWQ2> + (truncate:<VNARROWQ> + (SHIFTRT:VQN + (plus:VQN + (match_operand:VQN 2 "register_operand" "w") + (match_operand:VQN 4 "aarch64_int_rnd_operand")) + (match_operand:VQN 3 "aarch64_simd_shift_imm_vec_<vn_mode>"))) + (match_operand:<VNARROWQ> 1 "register_operand" "0")))] + "TARGET_SIMD && BYTES_BIG_ENDIAN + && aarch64_const_vec_rnd_cst_p (operands[4], operands[3])" + "rshrn2\t%<vn2>0.<V2ntype>, %<v>2.<Vtype>, %3" + [(set_attr "type" "neon_shift_imm_narrow_q")] +) + (define_expand "aarch64_<shrn_op>rshrn2_n<mode>" [(match_operand:<VNARROWQ2> 0 "register_operand") (match_operand:<VNARROWQ> 1 "register_operand") diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-1.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-1.c index 4bc3aa9563ee..a66b0e098289 100644 --- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-1.c +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-1.c @@ -10,6 +10,6 @@ uint8x16_t foo (uint32x4_t a, uint32x4_t b) return vrshrn_high_n_u16 (vrshrn_n_u16 (b1, 8), b1, 8); } -/* { dg-final { scan-assembler-times {\tmovi\t} 1 } } */ -/* { dg-final { scan-assembler-times {\traddhn\t} 2 } } */ -/* { dg-final { scan-assembler-times {\traddhn2\t} 2 } } */ +/* { dg-final { scan-assembler-not {\tmovi\t} } } */ +/* { dg-final { scan-assembler-times {\trshrn\t} 2 } } */ +/* { dg-final { scan-assembler-times {\trshrn2\t} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-2.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-2.c index 09d913e85524..6a3992832d89 100644 --- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-2.c +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-2.c @@ -1,4 +1,5 @@ /* { dg-do compile { target { aarch64*-*-* } } } */ +/* { dg-skip-if "" { *-*-* } { "-O0" } { "" } } */ #include <arm_neon.h> @@ -7,5 +8,5 @@ uint32x4_t foo (uint64x2_t a, uint64x2_t b) return vrshrn_high_n_u64 (vrshrn_n_u64 (a, 32), b, 32); } -/* { dg-final { scan-assembler-times {\traddhn\t} 1 } } */ -/* { dg-final { scan-assembler-times {\traddhn2\t} 1 } } */ +/* { dg-final { scan-assembler-times {\trshrn\t} 1 } } */ +/* { dg-final { scan-assembler-times {\trshrn2\t} 1 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-3.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-3.c index bdccbb3410f0..46c5f807e4f0 100644 --- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-3.c +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-3.c @@ -1,4 +1,5 @@ /* { dg-do compile { target { aarch64*-*-* } } } */ +/* { dg-skip-if "" { *-*-* } { "-O0" } { "" } } */ #include <arm_neon.h> @@ -7,5 +8,5 @@ uint16x8_t foo (uint32x4_t a, uint32x4_t b) return vrshrn_high_n_u32 (vrshrn_n_u32 (a, 16), b, 16); } -/* { dg-final { scan-assembler-times {\traddhn\t} 1 } } */ -/* { dg-final { scan-assembler-times {\traddhn2\t} 1 } } */ +/* { dg-final { scan-assembler-times {\trshrn\t} 1 } } */ +/* { dg-final { scan-assembler-times {\trshrn2\t} 1 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-4.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-4.c index 4b23eddb8589..10f31d045b62 100644 --- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-4.c +++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/shrn-4.c @@ -1,4 +1,5 @@ /* { dg-do compile { target { aarch64*-*-* } } } */ +/* { dg-skip-if "" { *-*-* } { "-O0" } { "" } } */ #include <arm_neon.h> @@ -7,5 +8,5 @@ uint8x16_t foo (uint16x8_t a, uint16x8_t b) return vrshrn_high_n_u16 (vrshrn_n_u16 (a, 8), b, 8); } -/* { dg-final { scan-assembler-times {\traddhn\t} 1 } } */ -/* { dg-final { scan-assembler-times {\traddhn2\t} 1 } } */ +/* { dg-final { scan-assembler-times {\trshrn\t} 1 } } */ +/* { dg-final { scan-assembler-times {\trshrn2\t} 1 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/simd_rshrn_1.c b/gcc/testsuite/gcc.target/aarch64/simd_rshrn_1.c new file mode 100644 index 000000000000..72a6af622ec0 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/simd_rshrn_1.c @@ -0,0 +1,106 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8-a" } */ +/* { dg-final { check-function-bodies "**" "" } } */ + +#include <stdint.h> + +typedef int16_t v8hi __attribute__((vector_size (16))); +typedef int8_t v8qi __attribute__((vector_size (8))); +typedef uint16_t v8uhi __attribute__((vector_size (16))); +typedef uint8_t v8uqi __attribute__((vector_size (8))); +typedef int32_t v4si __attribute__((vector_size (16))); +typedef int16_t v4hi __attribute__((vector_size (8))); +typedef uint32_t v4usi __attribute__((vector_size (16))); +typedef uint16_t v4uhi __attribute__((vector_size (8))); + +/* +** s16_s8: +** rshrn v0\.8b, v0\.8h, 6 +** ret +*/ +v8qi +s16_s8 (v8hi x) +{ + return __builtin_convertvector ((x + 32) >> 6, v8qi); +} + +/* +** u16_u8: +** rshrn v0\.8b, v0\.8h, 5 +** ret +*/ +v8uqi +u16_u8 (v8uhi x) +{ + return __builtin_convertvector ((x + 16) >> 5, v8uqi); +} + +/* +** s32_s16: +** rshrn v0\.4h, v0\.4s, 8 +** ret +*/ +v4hi +s32_s16 (v4si x) +{ + return __builtin_convertvector ((x + 128) >> 8, v4hi); +} + +/* +** u32_u16: +** rshrn v0\.4h, v0\.4s, 12 +** ret +*/ +v4uhi +u32_u16 (v4usi x) +{ + return __builtin_convertvector ((x + 2048) >> 12, v4uhi); +} + +#define ROUND_NARROW(NAME, TO, FROM, N, COUNT) \ + void \ + NAME (TO *__restrict d, const FROM *__restrict s) \ + { \ + for (int i = 0; i < COUNT; i++) \ + d[i] = (TO) ((s[i] + (1 << (N - 1))) >> N); \ + } + +/* +** loop_s16_s8: +** ldp q31, q30, \[x1\] +** rshrn v31\.8b, v31\.8h, 6 +** rshrn2 v31\.16b, v30\.8h, 6 +** str q31, \[x0\] +** ret +*/ +ROUND_NARROW (loop_s16_s8, int8_t, int16_t, 6, 16) + +/* +** loop_u16_u8: +** ldp q31, q30, \[x1\] +** rshrn v31\.8b, v31\.8h, 5 +** rshrn2 v31\.16b, v30\.8h, 5 +** str q31, \[x0\] +** ret +*/ +ROUND_NARROW (loop_u16_u8, uint8_t, uint16_t, 5, 16) + +/* +** loop_s32_s16: +** ldp q31, q30, \[x1\] +** rshrn v31\.4h, v31\.4s, 8 +** rshrn2 v31\.8h, v30\.4s, 8 +** str q31, \[x0\] +** ret +*/ +ROUND_NARROW (loop_s32_s16, int16_t, int32_t, 8, 8) + +/* +** loop_u32_u16: +** ldp q31, q30, \[x1\] +** rshrn v31\.4h, v31\.4s, 12 +** rshrn2 v31\.8h, v30\.4s, 12 +** str q31, \[x0\] +** ret +*/ +ROUND_NARROW (loop_u32_u16, uint16_t, uint32_t, 12, 8) diff --git a/gcc/testsuite/gcc.target/aarch64/simd_rshrn_2.c b/gcc/testsuite/gcc.target/aarch64/simd_rshrn_2.c new file mode 100644 index 000000000000..d056f753ddb9 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/simd_rshrn_2.c @@ -0,0 +1,64 @@ +/* { dg-do run } */ +/* { dg-options "-O3 -march=armv8-a" } */ + +#include <stdint.h> + +#define NELEMS 67 + +#define ROUND_NARROW(NAME, TO, FROM, N, DIV) \ + __attribute__((noipa)) static void \ + NAME (TO *__restrict d, const FROM *__restrict s, int n) \ + { \ + for (int i = 0; i < n; i++) \ + d[i] = (TO) ((s[i] + (1 << (N - 1))) >> N); \ + } \ + __attribute__((noipa, optimize ("O0"))) static void \ + NAME##_ref (TO *__restrict d, const FROM *__restrict s, int n) \ + { \ + for (int i = 0; i < n; i++) \ + d[i] = (TO) ((s[i] + (1 << (N - 1))) >> N); \ + } \ + static void \ + NAME##_check (void) \ + { \ + FROM s[NELEMS]; \ + TO d[NELEMS], r[NELEMS]; \ + for (int i = 0; i < NELEMS; i++) \ + s[i] = (FROM) rnd () / (DIV); \ + NAME (d, s, NELEMS); \ + NAME##_ref (r, s, NELEMS); \ + for (int i = 0; i < NELEMS; i++) \ + if (d[i] != r[i]) \ + __builtin_abort (); \ + } + +static uint64_t seed = 0x123456789abcdefULL; + +static uint64_t +rnd (void) +{ + seed = seed * 6364136223846793005ULL + 1442695040888963407ULL; + return seed >> 11; +} + +/* The signed 32-bit inputs are halved: signed overflow in the reference + addition would be undefined. The unsigned inputs are not halved, so the + wrapping add is covered. */ + +ROUND_NARROW (s16_s8, int8_t, int16_t, 6, 1) +ROUND_NARROW (u16_u8, uint8_t, uint16_t, 5, 1) +ROUND_NARROW (s32_s16, int16_t, int32_t, 8, 2) +ROUND_NARROW (u32_u16, uint16_t, uint32_t, 12, 1) + +int +main (void) +{ + for (int rep = 0; rep < 64; rep++) + { + s16_s8_check (); + u16_u8_check (); + s32_s16_check (); + u32_u16_check (); + } + return 0; +}
