https://gcc.gnu.org/g:b627bfa76649bb65ccab8d9aad9c1b1f1d45cdde
commit r17-4484-gb627bfa76649bb65ccab8d9aad9c1b1f1d45cdde Author: Kyrylo Tkachov <[email protected]> Date: Sun Aug 23 12:13:13 2026 +0200 match-sat-alu.pd: Recognize complement masks for unsigned saturating truncation Unsigned saturating truncation can form its sign mask from ~X instead of -X. This avoids negating the minimum signed value. For negative X, ~X is nonnegative and its top-bit shift is zero. For positive X above the result range, ~X is negative and its top-bit shift is all ones. Recognize this form. Keep the unsigned range comparison because a signed comparison does not clamp negative inputs. With AArch64 -O2 -march=armv8-a -fopenmp-simd: before: complement_clip4: ldr q31, [x1] movi v30.4s, 0xff, msl 8 not v29.16b, v31.16b cmhi v30.4s, v31.4s, v30.4s cmlt v29.4s, v29.4s, #0 bif v29.16b, v31.16b, v30.16b xtn v29.4h, v29.4s str d29, [x0] ret after: complement_clip4: ldr q31, [x1] movi v30.4s, 0 smax v30.4s, v31.4s, v30.4s uqxtn v30.4h, v30.4s str d30, [x0] ret Tests cover both widths, extrema, bounds, LTO, and a signed-comparison counterexample. The AArch64 test covers both valid forms. Bootstrapped and tested on aarch64-none-linux-gnu. gcc/ChangeLog: * match-sat-alu.pd (unsigned_integer_narrow_clip): Add the complement form. gcc/testsuite/ChangeLog: * gcc.dg/sat-trunc-signed-cmp-2.c: New test. * gcc.dg/vect/vect-sat-trunc-complement-1.c: Likewise. * gcc.target/aarch64/vect-sat-trunc-complement-1.c: Likewise. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/match-sat-alu.pd | 24 ++++++++ gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c | 42 +++++++++++++ .../gcc.dg/vect/vect-sat-trunc-complement-1.c | 70 ++++++++++++++++++++++ .../aarch64/vect-sat-trunc-complement-1.c | 30 ++++++++++ 4 files changed, 166 insertions(+) diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd index d8b4d9bb6b83..c79f8986ff50 100644 --- a/gcc/match-sat-alu.pd +++ b/gcc/match-sat-alu.pd @@ -301,6 +301,30 @@ along with GCC; see the file COPYING3. If not see && cmp >= 0 && expr_not_equal_to (@0, wi::min_value (itype_precision, SIGNED))))))) + (match (unsigned_integer_narrow_clip @0) + /* Use the sign of ~X for a source that avoids negating the minimum signed + value. For a negative X, ~X is nonnegative and gives zero. For a + positive X above the result range, ~X is negative and gives all ones. */ + (convert (cond^ (gt (nop_convert@3 @0) INTEGER_CST@1) + (rshift:s (bit_not @0) INTEGER_CST@2) + @0)) + (if (!TYPE_UNSIGNED (TREE_TYPE (@0)) + && TYPE_UNSIGNED (TREE_TYPE (@3))) + (with + { + unsigned itype_precision = TYPE_PRECISION (TREE_TYPE (@0)); + unsigned otype_precision = TYPE_PRECISION (type); + wide_int trunc_max = wi::mask (otype_precision, false, itype_precision); + wide_int int_cst_1 = wi::to_wide (@1, itype_precision); + wide_int int_cst_2 = wi::to_wide (@2, itype_precision); + wide_int shift_amount = wi::uhwi (itype_precision - 1, + itype_precision); + int cmp = wi::cmp (int_cst_2, shift_amount, + TYPE_SIGN (TREE_TYPE (@0))); + } + (if (otype_precision < itype_precision + && wi::eq_p (trunc_max, int_cst_1) + && cmp >= 0))))) (match (unsigned_integer_narrow_clip @0) /* SAT_US_TRUNC = (NT)MAX (MIN (X, NT_MAX), 0). */ (convert (integer_constant_clamp@3 @0 integer_zerop@1 @2)) diff --git a/gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c b/gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c new file mode 100644 index 000000000000..a06588ccedc0 --- /dev/null +++ b/gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c @@ -0,0 +1,42 @@ +/* The complement spelling of the narrow-clip idiom is only a clamp when the + comparison that selects the saturated arm is unsigned. With a signed + comparison a negative value takes the fall-through arm and keeps its low + bits, so it must not be turned into a saturating truncation. */ + +/* { dg-do run } */ +/* { dg-options "-O1 -ftree-vectorize -fdump-tree-vect-details" } */ + +typedef __UINT16_TYPE__ u16; +typedef __INT32_TYPE__ i32; + +#define N 32 + +static i32 a[N]; +static u16 r[N]; + +__attribute__((noipa)) static void +clip (u16 *__restrict d, const i32 *__restrict s, int n) +{ + for (int i = 0; i < n; i++) + { + i32 v = s[i]; + d[i] = v > 65535 ? (~v) >> 31 : v; + } +} + +int +main (void) +{ + for (int i = 0; i < N; i++) + a[i] = -1; + + clip (r, a, N); + + for (int i = 0; i < N; i++) + if (r[i] != (u16) -1) + __builtin_abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-complement-1.c b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-complement-1.c new file mode 100644 index 000000000000..f7bb1656a5d4 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-complement-1.c @@ -0,0 +1,70 @@ +/* { dg-do run } */ +/* { dg-require-effective-target vect_int } */ +/* { dg-additional-options "-O3 -fvect-cost-model=unlimited" } */ + +#include "tree-vect.h" + +typedef __UINT16_TYPE__ u16; +typedef __UINT32_TYPE__ u32; +typedef __INT32_TYPE__ i32; +typedef __INT64_TYPE__ i64; + +#define N 96 + +static i32 in32[N]; +static i64 in64[N]; +static u16 out16[N]; +static u32 out32[N]; + +__attribute__((noipa)) static void +clip_u16 (u16 *__restrict out, const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + out[i] = (x & ~65535) ? (~x) >> 31 : x; + } +} + +__attribute__((noipa)) static void +clip_u32 (u32 *__restrict out, const i64 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i64 x = in[i]; + out[i] = (x & ~(i64) 0xffffffff) ? (~x) >> 63 : x; + } +} + +int +main (void) +{ + static const i32 values32[6] = + { (-2147483647 - 1), -1, 0, 65535, 65536, 2147483647 }; + static const u16 expected16[6] = + { 0, 0, 0, 65535, 65535, 65535 }; + static const i64 values64[6] = + { (-9223372036854775807LL - 1), -1, 0, 4294967295LL, + 4294967296LL, 9223372036854775807LL }; + static const u32 expected32[6] = + { 0, 0, 0, (u32) -1, (u32) -1, (u32) -1 }; + + check_vect (); + + for (int i = 0; i < N; ++i) + { + in32[i] = values32[i % 6]; + in64[i] = values64[i % 6]; + } + + clip_u16 (out16, in32, N); + clip_u32 (out32, in64, N); + +#pragma GCC novector + for (int i = 0; i < N; ++i) + if (out16[i] != expected16[i % 6] + || out32[i] != expected32[i % 6]) + __builtin_abort (); + + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-complement-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-complement-1.c new file mode 100644 index 000000000000..9c18ee063e00 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-complement-1.c @@ -0,0 +1,30 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8-a" } */ +/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */ + +typedef __UINT16_TYPE__ u16; +typedef __UINT32_TYPE__ u32; +typedef __INT32_TYPE__ i32; +typedef __INT64_TYPE__ i64; + +void +clip_u16 (u16 *__restrict out, const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + out[i] = (x & ~65535) ? (~x) >> 31 : x; + } +} + +void +clip_u32 (u32 *__restrict out, const i64 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i64 x = in[i]; + out[i] = (x & ~(i64) 4294967295LL) ? (~x) >> 63 : x; + } +} + +/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 2 "vect" } } */
