https://gcc.gnu.org/g:28f6899c51c456302943d549ab78d9bb022f36d8
commit r17-4076-g28f6899c51c456302943d549ab78d9bb022f36d8 Author: Kyrylo Tkachov <[email protected]> Date: Sat Sep 5 12:55:03 2026 +0200 match-sat-alu.pd: Recognize complement tests for unsigned saturating addition For an unsigned type, MAX - Y is ~Y. An overflow test can therefore use X > ~Y ? MAX : X + Y Recognize strict and non-strict comparisons with either operand or result arm order. Equality is safe because both arms produce MAX. Also recognize ADD_OVERFLOW forms that select a separate X + Y expression. The comparison iterators cover strict and non-strict forms. The `:c` modifier covers both operand orders. AArch64 -O3, vector body: before: add v29.4s, v31.4s, v30.4s not v30.16b, v30.16b cmhs v30.4s, v30.4s, v31.4s orn v30.16b, v29.16b, v30.16b after: uqadd v30.4s, v31.4s, v30.4s Four vector operations become one. Bootstrapped and regression tested on aarch64-unknown-linux-gnu. gcc/ChangeLog: * match-sat-alu.pd (unsigned_integer_sat_add): Recognise comparisons against the complement of an operand. Also recognise ADD_OVERFLOW with a separate addition. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sat_u_add_not_cmp-1.c: New test. * gcc.target/aarch64/sat_u_add_not_cmp-2.c: New test. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/match-sat-alu.pd | 22 ++++++++++++++++ .../gcc.target/aarch64/sat_u_add_not_cmp-1.c | 30 ++++++++++++++++++++++ .../gcc.target/aarch64/sat_u_add_not_cmp-2.c | 16 ++++++++++++ 3 files changed, 68 insertions(+) diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd index 83cd8963147c..e787a7414f43 100644 --- a/gcc/match-sat-alu.pd +++ b/gcc/match-sat-alu.pd @@ -104,6 +104,28 @@ along with GCC; see the file COPYING3. If not see (cond^ (ne (imagpart (IFN_ADD_OVERFLOW@2 @0 INTEGER_CST@1)) integer_zerop) integer_minus_onep (realpart @2)) (if (types_match (type, @0) && int_fits_type_p (@1, type)))) + (match (unsigned_integer_sat_add @0 @1) + /* SUM = ADD_OVERFLOW (X, Y) + SAT_U_ADD = IMAGPART (SUM) == 0 ? X + Y : -1. */ + (cond^ (eq (imagpart (IFN_ADD_OVERFLOW @0 @1)) integer_zerop) + (plus @0 @1) integer_minus_onep)) + (match (unsigned_integer_sat_add @0 @1) + /* SUM = ADD_OVERFLOW (X, Y) + SAT_U_ADD = IMAGPART (SUM) != 0 ? -1 : X + Y. */ + (cond^ (ne (imagpart (IFN_ADD_OVERFLOW @0 @1)) integer_zerop) + integer_minus_onep (plus @0 @1))) + /* A source that tests for overflow against the complement of an operand, + such as a > MAX - b ? MAX : a + b, reaches the recognisers as a compare + with ~Y, since ~Y is MAX - Y. The strict and non-strict forms are both + exact because at X == ~Y the sum is MAX and the two arms agree. */ + (for cmp (ge gt) + (match (unsigned_integer_sat_add @0 @1) + (cond^ (cmp:c (bit_not @1) @0) + (plus:c @0 @1) integer_minus_onep))) + (for cmp (lt le) + (match (unsigned_integer_sat_add @0 @1) + (cond^ (cmp:c (bit_not @1) @0) + integer_minus_onep (plus:c @0 @1)))) (match (unsigned_integer_sat_add @0 @1) /* WIDEN_SUM = (WT)X + (WT)Y SAT_U_ADD = WIDEN_SUM > MAX ? MAX : (NT)WIDEN_SUM */ diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-1.c b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-1.c new file mode 100644 index 000000000000..88041f53f614 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-1.c @@ -0,0 +1,30 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fdump-tree-optimized" } */ + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned long long u64; + +/* Every one of these tests for overflow against MAX - b, which reaches the + middle end as a comparison with ~b. The strict and the non-strict form + agree, because at a == MAX - b the sum is MAX and so is the other arm. */ + +#define DEF(N, T, MX) \ + T f1_##N (T a, T b) { return a > (T) (MX - b) ? MX : a + b; } \ + T f2_##N (T a, T b) { return a >= (T) (MX - b) ? MX : a + b; } \ + T f3_##N (T a, T b) { return b > (T) ~a ? MX : a + b; } \ + T f4_##N (T a, T b) { return b >= (T) ~a ? MX : a + b; } \ + T f5_##N (T a, T b) { return a <= (T) (MX - b) ? a + b : MX; } \ + T f6_##N (T a, T b) { return a < (T) (MX - b) ? a + b : MX; } \ + T f7_##N (T a, T b) { return (T) (MX - a) < b ? MX : a + b; } \ + T f8_##N (T a, T b) { return (T) (MX - a) <= b ? MX : a + b; } \ + T f9_##N (T a, T b) { return (T) ~b >= a ? a + b : MX; } \ + T f10_##N (T a, T b) { return (T) ~b > a ? a + b : MX; } + +DEF (8, u8, 255) +DEF (16, u16, 65535) +DEF (32, u32, 0xffffffffu) +DEF (64, u64, ~0ull) + +/* { dg-final { scan-tree-dump-times "\\.SAT_ADD " 40 "optimized" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-2.c b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-2.c new file mode 100644 index 000000000000..bcd81304f91c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-2.c @@ -0,0 +1,16 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -fdump-tree-optimized" } */ + +typedef unsigned int u32; + +/* Recognition lets the loop vectorize to one uqadd. */ + +void +f (u32 *__restrict d, u32 *__restrict a, u32 *__restrict b, int n) +{ + for (int i = 0; i < n; i++) + d[i] = a[i] > 0xffffffffu - b[i] ? 0xffffffffu : a[i] + b[i]; +} + +/* { dg-final { scan-tree-dump "\\.SAT_ADD " "optimized" } } */ +/* { dg-final { scan-assembler "uqadd\tv\[0-9\]+\.4s" } } */
