https://gcc.gnu.org/g:7cbb7185d07fca6f087a7426e5ad4c402a08c366
commit r17-3900-g7cbb7185d07fca6f087a7426e5ad4c402a08c366 Author: Kyrylo Tkachov <[email protected]> Date: Thu Sep 3 15:44:33 2026 +0200 aarch64: Use SVE2 SQSHRNB and UQSHRNB for narrowing shifts A constant right shift feeding a saturating truncation stays as a separate shift, although SVE2 has SQSHRNB and UQSHRNB for the fused operation. Match the saturating truncation around an arithmetic or logical right shift by a constant vector, over the same mode pairs as the plain narrowing. Extend vn_mode to those modes so that it selects the shift predicate, which bounds the amount by the precision of the result, and that is also the range the instructions accept. For void f (uint8_t *__restrict d, const uint16_t *__restrict s, int n) { for (int i = 0; i < n; ++i) { uint16_t x = s[i] >> 7; d[i] = x > 255 ? 255 : x; } } the inner loop changes from ld1h z31.h, p7/z, [x1, x3, lsl 1] lsr z31.h, z31.h, #7 umin z31.h, z31.h, #255 st1b z31.h, p7, [x0, x3] to ld1h z31.h, p7/z, [x1, x3, lsl 1] uqshrnb z31.b, z31.h, #7 st1b z31.h, p7, [x0, x3] Bootstrapped and tested on aarch64-none-linux-gnu. gcc/ChangeLog: * config/aarch64/iterators.md (vn_mode): Add the SVE modes whose elements halve without changing the number of lanes. * config/aarch64/aarch64-sve2.md (*aarch64_sve_<shrn_op>shrnb<mode>): New pattern. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sve2/vect-sat-trunc-shift-1.c: New test. * gcc.target/aarch64/sve2/vect-sat-trunc-shift-1_run.c: Likewise. * gcc.target/aarch64/sve2/vect-sat-trunc-shift-2.c: Likewise. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/config/aarch64/aarch64-sve2.md | 15 ++++++ gcc/config/aarch64/iterators.md | 4 +- .../aarch64/sve2/vect-sat-trunc-shift-1.c | 57 ++++++++++++++++++++ .../aarch64/sve2/vect-sat-trunc-shift-1_run.c | 60 ++++++++++++++++++++++ .../aarch64/sve2/vect-sat-trunc-shift-2.c | 36 +++++++++++++ 5 files changed, 171 insertions(+), 1 deletion(-) diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index 9e55781f10f5..c33df283cc2a 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -3202,6 +3202,21 @@ ;; - UQSHRNT ;; ------------------------------------------------------------------------- +;; Fold a constant right shift into the saturating narrowing. The predicate +;; bounds the shift amount by the precision of the result, which is the range +;; the instructions accept. +(define_insn "*aarch64_sve_<shrn_op>shrnb<mode>" + [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w") + (SAT_TRUNC:<VNARROWQ> + (<TRUNC_SHIFT>:SVE_HSDI + (match_operand:SVE_HSDI 1 "register_operand" "w") + (match_operand:SVE_HSDI 2 + "aarch64_simd_shift_imm_vec_<vn_mode>"))))] + "TARGET_SVE2" + "<shrn_op>shrnb\t%0.<Ventype>, %1.<Vetype>, #%2" + [(set_attr "sve_type" "sve_int_shift")] +) + ;; The immediate range is enforced before generating the instruction. (define_insn "@aarch64_sve_<sve_int_op><mode>" [(set (match_operand:<VNARROW> 0 "register_operand" "=w") diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index 0c69c2148dce..7b7c2bc34e55 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -2555,7 +2555,9 @@ ;; Like ve_mode but for the half-width modes. (define_mode_attr vn_mode [(V8HI "qi") (V4SI "hi") (V2DI "si") (DI "si") - (SI "hi") (HI "qi")]) + (SI "hi") (HI "qi") + (VNx8HI "qi") (VNx4HI "qi") (VNx2HI "qi") + (VNx4SI "hi") (VNx2SI "hi") (VNx2DI "si")]) ;; Vm for lane instructions is restricted to FP_LO_REGS. (define_mode_attr vwx [(V4HI "x") (V8HI "x") (HI "x") diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-1.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-1.c new file mode 100644 index 000000000000..2ee8b90a9b79 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-1.c @@ -0,0 +1,57 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -mautovec-preference=sve-only" } */ + +typedef __INT8_TYPE__ int8_t; +typedef __INT16_TYPE__ int16_t; +typedef __INT32_TYPE__ int32_t; +typedef __INT64_TYPE__ int64_t; +typedef __UINT8_TYPE__ uint8_t; +typedef __UINT16_TYPE__ uint16_t; +typedef __UINT32_TYPE__ uint32_t; +typedef __UINT64_TYPE__ uint64_t; + +/* The largest shift amounts for which the saturation survives value range + propagation. A larger amount makes the clamp dead, so the fused form is + unreachable from C. */ +#define SHIFT_H 7 +#define SHIFT_S 15 +#define SHIFT_D 31 + +#define DEF_UNSIGNED(NAME, OUT, IN, SHIFT, MAX) \ + void __attribute__((noipa)) \ + NAME (OUT *__restrict out, const IN *__restrict in, int n) \ + { \ + for (int i = 0; i < n; ++i) \ + { \ + IN x = in[i] >> SHIFT; \ + out[i] = x > (IN) MAX ? (OUT) MAX : (OUT) x; \ + } \ + } + +#define DEF_SIGNED(NAME, OUT, IN, SHIFT, MIN, MAX) \ + void __attribute__((noipa)) \ + NAME (OUT *__restrict out, const IN *__restrict in, int n) \ + { \ + for (int i = 0; i < n; ++i) \ + { \ + IN x = in[i] >> SHIFT; \ + OUT tmp = (OUT) x; \ + out[i] = ((IN) MIN <= x && x <= (IN) MAX \ + ? tmp : x < 0 ? (OUT) MIN : (OUT) MAX); \ + } \ + } + +DEF_UNSIGNED (u16_to_u8, uint8_t, uint16_t, SHIFT_H, 255) +DEF_UNSIGNED (u32_to_u16, uint16_t, uint32_t, SHIFT_S, 65535) +DEF_UNSIGNED (u64_to_u32, uint32_t, uint64_t, SHIFT_D, 4294967295ULL) +DEF_SIGNED (s16_to_s8, int8_t, int16_t, SHIFT_H, -128, 127) +DEF_SIGNED (s32_to_s16, int16_t, int32_t, SHIFT_S, -32768, 32767) +DEF_SIGNED (s64_to_s32, int32_t, int64_t, SHIFT_D, -2147483647 - 1, 2147483647) + +/* { dg-final { scan-assembler-times {\tuqshrnb\tz[0-9]+\.b, z[0-9]+\.h, #7\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tuqshrnb\tz[0-9]+\.h, z[0-9]+\.s, #15\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tuqshrnb\tz[0-9]+\.s, z[0-9]+\.d, #31\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsqshrnb\tz[0-9]+\.b, z[0-9]+\.h, #7\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsqshrnb\tz[0-9]+\.h, z[0-9]+\.s, #15\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tsqshrnb\tz[0-9]+\.s, z[0-9]+\.d, #31\n} 1 } } */ +/* { dg-final { scan-assembler-not {\t[lsa]sr\tz} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-1_run.c new file mode 100644 index 000000000000..c39eb222ff82 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-1_run.c @@ -0,0 +1,60 @@ +/* { dg-do run } */ +/* { dg-require-effective-target aarch64_sve2_hw } */ +/* { dg-options "-O3 -mautovec-preference=sve-only" } */ + +#include "vect-sat-trunc-shift-1.c" + +/* Not a multiple of any SVE vector length, so that the loops run a + predicated tail iteration. */ +#define N 257 + +#define INIT(IN, ARR) \ + for (int i = 0; i < N; ++i) \ + (ARR)[i] = (i & 1) ? (IN) (i - N / 2) \ + : (IN) (i * 0x9e3779b97f4a7c15ULL) + +#define CHECK_UNSIGNED(NAME, OUT, IN, SHIFT, MAX) \ + do { \ + IN in[N]; \ + OUT out[N]; \ + INIT (IN, in); \ + NAME (out, in, N); \ + _Pragma ("GCC novector") \ + for (int i = 0; i < N; ++i) \ + { \ + IN x = in[i] >> SHIFT; \ + if (out[i] != (OUT) (x > (IN) MAX ? (IN) MAX : x)) \ + __builtin_abort (); \ + } \ + } while (0) + +#define CHECK_SIGNED(NAME, OUT, IN, SHIFT, MIN, MAX) \ + do { \ + IN in[N]; \ + OUT out[N]; \ + INIT (IN, in); \ + NAME (out, in, N); \ + _Pragma ("GCC novector") \ + for (int i = 0; i < N; ++i) \ + { \ + IN x = in[i] >> SHIFT; \ + OUT tmp = (OUT) x; \ + OUT ref = ((IN) MIN <= x && x <= (IN) MAX \ + ? tmp : x < 0 ? (OUT) MIN : (OUT) MAX); \ + if (out[i] != ref) \ + __builtin_abort (); \ + } \ + } while (0) + +int +main (void) +{ + CHECK_UNSIGNED (u16_to_u8, uint8_t, uint16_t, SHIFT_H, 255); + CHECK_UNSIGNED (u32_to_u16, uint16_t, uint32_t, SHIFT_S, 65535); + CHECK_UNSIGNED (u64_to_u32, uint32_t, uint64_t, SHIFT_D, 4294967295ULL); + CHECK_SIGNED (s16_to_s8, int8_t, int16_t, SHIFT_H, -128, 127); + CHECK_SIGNED (s32_to_s16, int16_t, int32_t, SHIFT_S, -32768, 32767); + CHECK_SIGNED (s64_to_s32, int32_t, int64_t, SHIFT_D, -2147483647 - 1, + 2147483647); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-2.c b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-2.c new file mode 100644 index 000000000000..c66e80ec5835 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/vect-sat-trunc-shift-2.c @@ -0,0 +1,36 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -mautovec-preference=sve-only -fdump-tree-vect-details" } */ + +typedef __UINT8_TYPE__ uint8_t; +typedef __UINT16_TYPE__ uint16_t; +typedef __UINT32_TYPE__ uint32_t; + +/* A variable shift amount has no immediate form, so the shift stays + separate from the saturating narrowing. */ + +void __attribute__((noipa)) +h_to_b (uint8_t *__restrict out, const uint16_t *__restrict in, + const uint16_t *__restrict shifts, int n) +{ + for (int i = 0; i < n; ++i) + { + uint16_t x = in[i] >> (shifts[i] & 15); + out[i] = x > 255 ? 255 : x; + } +} + +void __attribute__((noipa)) +s_to_h (uint16_t *__restrict out, const uint32_t *__restrict in, + const uint32_t *__restrict shifts, int n) +{ + for (int i = 0; i < n; ++i) + { + uint32_t x = in[i] >> (shifts[i] & 31); + out[i] = x > 65535 ? 65535 : x; + } +} + +/* { dg-final { scan-assembler-not {\t[su]qshrnb\t} } } */ +/* { dg-final { scan-assembler-times {\tuqxtnb\tz[0-9]+\.b, z[0-9]+\.h\n} 1 } } */ +/* { dg-final { scan-assembler-times {\tuqxtnb\tz[0-9]+\.h, z[0-9]+\.s\n} 1 } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loops in function" 2 "vect" } } */
