https://gcc.gnu.org/g:02b0226f9f8ac1face83c096b6e783e5c8956f6a
commit r17-4482-g02b0226f9f8ac1face83c096b6e783e5c8956f6a Author: Kyrylo Tkachov <[email protected]> Date: Mon Sep 14 09:24:22 2026 +0200 match-sat-alu.pd: Recognize unsigned constant-clamp truncation A signed value clamped to the range of a narrower unsigned type is unsigned saturating truncation. Recognize nested MIN and MAX operations in either order. Keep the clamp match independent of use count. At this use, require an exact zero lower bound, the maximum upper bound, and one use of the outer clamp. The inner clamp can have other uses. With AArch64 -O2 -march=armv8-a -fopenmp-simd: before: clamp_u16_4: ldr q0, [x1] movi v30.4s, 0xff, msl 8 movi v31.4s, 0 smin v30.4s, v0.4s, v30.4s smax v30.4s, v30.4s, v31.4s xtn v30.4h, v30.4s str d30, [x0] ret after: clamp_u16_4: ldr q31, [x1] movi v30.4s, 0 smax v30.4s, v31.4s, v30.4s uqxtn v30.4h, v30.4s str d30, [x0] ret Runtime tests cover both clamp orders, two widths, and LTO. Target tests cover valid bounds, invalid bounds, and extra uses of each clamp. Bootstrapped and tested on aarch64-none-linux-gnu. gcc/ChangeLog: * match-sat-alu.pd (integer_constant_clamp): New match. (unsigned_integer_narrow_clip): Add the constant-clamp form. gcc/testsuite/ChangeLog: * gcc.dg/vect/vect-sat-trunc-clamp-1.c: New test. * gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c: Likewise. * gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c: Likewise. * gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c: Likewise. * gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c: Likewise. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/match-sat-alu.pd | 23 +++- gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c | 121 +++++++++++++++++++++ .../aarch64/vect-sat-trunc-clamp-inner-use-1.c | 22 ++++ .../aarch64/vect-sat-trunc-clamp-multi-use-1.c | 21 ++++ .../gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c | 7 ++ .../aarch64/vect-sat-trunc-clamp-u-negative-1.c | 30 +++++ 6 files changed, 223 insertions(+), 1 deletion(-) diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd index 5a5ed52d40de..c4b24dfab11a 100644 --- a/gcc/match-sat-alu.pd +++ b/gcc/match-sat-alu.pd @@ -23,6 +23,13 @@ along with GCC; see the file COPYING3. If not see <http://www.gnu.org/licenses/>. */ +/* Clamp @0 between the constant bounds @1 and @2. The source can put the + MIN or the MAX first. */ +(match (integer_constant_clamp @0 @1 @2) + (max (min @0 INTEGER_CST@2) INTEGER_CST@1)) +(match (integer_constant_clamp @0 @1 @2) + (min (max @0 INTEGER_CST@1) INTEGER_CST@2)) + /* Saturation add for unsigned integer. */ (if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type)) (match (usadd_overflow_mask @0 @1) @@ -293,7 +300,21 @@ along with GCC; see the file COPYING3. If not see && wi::eq_p (trunc_max, int_cst_1) && cmp >= 0 && expr_not_equal_to (@0, wi::min_value (itype_precision, - SIGNED)))))))) + SIGNED))))))) + (match (unsigned_integer_narrow_clip @0) + /* SAT_US_TRUNC = (NT)MAX (MIN (X, NT_MAX), 0). */ + (convert (integer_constant_clamp@3 @0 integer_zerop@1 @2)) + (if (!TYPE_UNSIGNED (TREE_TYPE (@0)) + && single_use (@3)) + (with + { + unsigned itype_precision = TYPE_PRECISION (TREE_TYPE (@0)); + unsigned otype_precision = TYPE_PRECISION (type); + wide_int trunc_max = wi::mask (otype_precision, false, itype_precision); + wide_int hi_cst = wi::to_wide (@2, itype_precision); + } + (if (otype_precision < itype_precision + && wi::eq_p (hi_cst, trunc_max))))))) /* Saturation truncate for unsigned integer. */ (if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type)) diff --git a/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c new file mode 100644 index 000000000000..06a4428d0d24 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c @@ -0,0 +1,121 @@ +/* { dg-do run } */ +/* { dg-require-effective-target vect_int } */ +/* { dg-additional-options "-O3 -fvect-cost-model=unlimited" } */ + +#include "tree-vect.h" + +typedef __UINT16_TYPE__ u16; +typedef __UINT32_TYPE__ u32; +typedef __UINT64_TYPE__ u64; +typedef __INT32_TYPE__ i32; +typedef __INT64_TYPE__ i64; + +#define N 259 + +static i32 in32[N]; +static i64 in64[N]; +static u16 out16[N]; +static u32 out32[N]; + +__attribute__((noipa)) static void +clamp_u16_lo (u16 *__restrict out, const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + out[i] = x < 0 ? 0 : (x > 65535 ? 65535 : x); + } +} + +__attribute__((noipa)) static void +clamp_u16_hi (u16 *__restrict out, const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + out[i] = x > 65535 ? 65535 : (x < 0 ? 0 : x); + } +} + +__attribute__((noipa)) static void +clamp_u32_lo (u32 *__restrict out, const i64 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i64 x = in[i]; + out[i] = x < 0 ? 0 : (x > 4294967295LL ? 4294967295LL : x); + } +} + +__attribute__((noipa)) static void +clamp_u32_hi (u32 *__restrict out, const i64 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i64 x = in[i]; + out[i] = x > 4294967295LL ? 4294967295LL : (x < 0 ? 0 : x); + } +} + +static u16 +ref_u16 (i32 x) +{ + return x < 0 ? 0 : (x > 65535 ? 65535 : x); +} + +static u32 +ref_u32 (i64 x) +{ + return x < 0 ? 0 : (x > 4294967295LL ? 4294967295LL : x); +} + +static void +check_u16 (void) +{ +#pragma GCC novector + for (int i = 0; i < N; ++i) + if (out16[i] != ref_u16 (in32[i])) + __builtin_abort (); +} + +static void +check_u32 (void) +{ +#pragma GCC novector + for (int i = 0; i < N; ++i) + if (out32[i] != ref_u32 (in64[i])) + __builtin_abort (); +} + +int +main (void) +{ + check_vect (); + + for (int i = 0; i < N; ++i) + { + in32[i] = (i32) ((u32) i * 2654435761U + 1013904223U); + in64[i] = (i64) ((u64) i * 11400714819323198485ULL + + 13787848793156543929ULL); + } + + in32[0] = -1; + in32[1] = 0; + in32[2] = 65535; + in32[3] = 65536; + in64[0] = -1; + in64[1] = 0; + in64[2] = 4294967295LL; + in64[3] = 4294967296LL; + + clamp_u16_lo (out16, in32, N); + check_u16 (); + clamp_u16_hi (out16, in32, N); + check_u16 (); + clamp_u32_lo (out32, in64, N); + check_u32 (); + clamp_u32_hi (out32, in64, N); + check_u32 (); + + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c new file mode 100644 index 000000000000..b93bc56188e6 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c @@ -0,0 +1,22 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8-a" } */ +/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */ + +typedef __UINT16_TYPE__ u16; +typedef __INT32_TYPE__ i32; + +void +clip (u16 *__restrict out, i32 *__restrict copy, + const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + i32 inner = x > 65535 ? 65535 : x; + i32 outer = inner < 0 ? 0 : inner; + out[i] = outer; + copy[i] = inner; + } +} + +/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c new file mode 100644 index 000000000000..0b82d40ff2f9 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8-a" } */ +/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */ + +typedef __UINT16_TYPE__ u16; +typedef __INT32_TYPE__ i32; + +void +clip (u16 *__restrict out, i32 *__restrict copy, + const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + i32 clamped = x < 0 ? 0 : (x > 65535 ? 65535 : x); + out[i] = clamped; + copy[i] = clamped; + } +} + +/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c new file mode 100644 index 000000000000..f159a8f4dac7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c @@ -0,0 +1,7 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8-a" } */ +/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */ + +#include "../../gcc.dg/vect/vect-sat-trunc-clamp-1.c" + +/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 4 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c new file mode 100644 index 000000000000..aa283f176ad8 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c @@ -0,0 +1,30 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8-a" } */ +/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */ + +typedef __UINT16_TYPE__ u16; +typedef __INT32_TYPE__ i32; + +__attribute__((noipa)) +void +bad_low (u16 *__restrict out, const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + out[i] = x < 1 ? 1 : (x > 65535 ? 65535 : x); + } +} + +__attribute__((noipa)) +void +bad_high (u16 *__restrict out, const i32 *__restrict in, int n) +{ + for (int i = 0; i < n; ++i) + { + i32 x = in[i]; + out[i] = x > 65534 ? 65534 : (x < 0 ? 0 : x); + } +} + +/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */
