https://gcc.gnu.org/g:1320bc377dab63d1c9a83662c49d01f3a166332c
commit r17-4218-g1320bc377dab63d1c9a83662c49d01f3a166332c Author: Kyrylo Tkachov <[email protected]> Date: Thu Jul 30 18:13:03 2026 +0200 match-sat-alu.pd: Recognize X - MIN (X, Y) as saturating subtraction X - MIN (X, Y) is unsigned saturating subtraction, but subtraction statements were not checked for this form. Recognize the form and check MINUS_EXPR statements for saturation. Require the MIN to have one use so that the replacement removes it. Make the assign recognizers report whether they replaced the statement, and do not run later transforms after a replacement. AArch64 -O2: before: cmp w1, w0 csel w1, w1, w0, ls sub w0, w0, w1 after: subs w0, w0, w1 csel w0, w0, wzr, cs The vector form becomes one UQSUB instead of a comparison, a select, and a subtraction. Bootstrapped and tested on aarch64-none-linux-gnu. gcc/ChangeLog: * match-sat-alu.pd (unsigned_integer_sat_sub): Add the X - MIN (X, Y) form. * tree-ssa-math-opts.cc (build_saturation_binary_arith_call_and_replace): Return whether a replacement was made. (match_saturation_add_with_assign): Likewise. (match_unsigned_saturation_sub): Likewise. (math_opts_dom_walker::after_dom_children): Check MINUS_EXPR for saturation, and use the assign recognizer results. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sat_u_sub_minmax-1.c: New test. * gcc.target/aarch64/sat_u_sub_minmax-2.c: New test. * gcc.target/aarch64/sat_u_sub_minmax-3.c: New test. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/match-sat-alu.pd | 6 ++++ .../gcc.target/aarch64/sat_u_sub_minmax-1.c | 18 +++++++++++ .../gcc.target/aarch64/sat_u_sub_minmax-2.c | 32 +++++++++++++++++++ .../gcc.target/aarch64/sat_u_sub_minmax-3.c | 17 ++++++++++ gcc/tree-ssa-math-opts.cc | 36 ++++++++++++++-------- 5 files changed, 97 insertions(+), 12 deletions(-) diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd index e787a7414f43..45f74ca5ace8 100644 --- a/gcc/match-sat-alu.pd +++ b/gcc/match-sat-alu.pd @@ -169,6 +169,12 @@ along with GCC; see the file COPYING3. If not see (match (unsigned_integer_sat_sub @0 @1) /* SAT_U_SUB = (X - Y) <= X ? X - Y : 0, and the commuted comparison. */ (cond^ (le:c (minus@2 @0 @1) @0) @2 integer_zerop)) + (match (unsigned_integer_sat_sub @0 @1) + /* SAT_U_SUB = X - MIN (X, Y). The MIN has to be single use: while it + stays live the saturating subtract is computed beside it instead of + replacing it, which costs an instruction. */ + (minus @0 (min:c@2 @0 @1)) + (if (single_use (@2)))) (match (unsigned_integer_sat_sub @0 @1) /* DIFF = SUB_OVERFLOW (X, Y) SAT_U_SUB = REALPART (DIFF) | (IMAGPART (DIFF) + (-1)) */ diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-1.c b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-1.c new file mode 100644 index 000000000000..aa1d7b6f0315 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-1.c @@ -0,0 +1,18 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fdump-tree-optimized" } */ + +#define DEF_MIN(T) \ + T min_##T (T a, T b) { return a - (a < b ? a : b); } \ + T nim_##T (T a, T b) { return a - (b < a ? b : a); } + +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned long long u64; + +DEF_MIN (u8) +DEF_MIN (u16) +DEF_MIN (u32) +DEF_MIN (u64) + +/* { dg-final { scan-tree-dump-times "\\.SAT_SUB " 8 "optimized" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-2.c b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-2.c new file mode 100644 index 000000000000..dcf8ab463099 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-2.c @@ -0,0 +1,32 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -fdump-tree-optimized" } */ + +typedef unsigned char u8; +typedef unsigned int u32; + +/* The recognition must reach the vectoriser, so that the loop becomes a + single uqsub rather than a compare, a select and a subtraction. */ + +void +min_loop (u8 *__restrict d, u8 *__restrict a, u8 *__restrict b, int n) +{ + for (int i = 0; i < n; i++) + { + u8 x = a[i], y = b[i]; + d[i] = x - (x < y ? x : y); + } +} + +void +min_loop32 (u32 *__restrict d, u32 *__restrict a, u32 *__restrict b, int n) +{ + for (int i = 0; i < n; i++) + { + u32 x = a[i], y = b[i]; + d[i] = x - (y < x ? y : x); + } +} + +/* { dg-final { scan-tree-dump "\\.SAT_SUB " "optimized" } } */ +/* { dg-final { scan-assembler "uqsub\tv\[0-9\]+\\.16b" } } */ +/* { dg-final { scan-assembler "uqsub\tv\[0-9\]+\\.4s" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-3.c b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-3.c new file mode 100644 index 000000000000..ca71343699d9 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-3.c @@ -0,0 +1,17 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fdump-tree-optimized" } */ + +typedef unsigned int u32; + +/* While the MIN stays live the saturating subtract would be computed beside + it rather than instead of it, so the rules do not fire. */ + +u32 +min_live (u32 a, u32 b, u32 *o) +{ + u32 m = a < b ? a : b; + *o = m; + return a - m; +} + +/* { dg-final { scan-tree-dump-not "\\.SAT_SUB " "optimized" } } */ diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc index 70ac17c9e2fa..9c959cc84508 100644 --- a/gcc/tree-ssa-math-opts.cc +++ b/gcc/tree-ssa-math-opts.cc @@ -4123,7 +4123,7 @@ extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree)); extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree)); extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree)); -static void +static bool build_saturation_binary_arith_call_and_replace (gimple_stmt_iterator *gsi, internal_fn fn, tree lhs, tree op_0, tree op_1) @@ -4133,7 +4133,10 @@ build_saturation_binary_arith_call_and_replace (gimple_stmt_iterator *gsi, gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1); gimple_call_set_lhs (call, lhs); gsi_replace (gsi, call, /* update_eh_info */ true); + return true; } + + return false; } static bool @@ -4177,9 +4180,10 @@ build_saturation_binary_arith_call_and_insert (gimple_stmt_iterator *gsi, * # _9 = PHI <_2(3), 128(2)> * _4 = (int8_t) _9; * => - * _4 = .SAT_ADD (x_5, -1); */ + * _4 = .SAT_ADD (x_5, -1); + * Return true if the statement was replaced. */ -static void +static bool match_saturation_add_with_assign (gimple_stmt_iterator *gsi, gassign *stmt) { tree ops[2]; @@ -4187,8 +4191,10 @@ match_saturation_add_with_assign (gimple_stmt_iterator *gsi, gassign *stmt) if (gimple_unsigned_integer_sat_add (lhs, ops, NULL) || gimple_signed_integer_sat_add (lhs, ops, NULL)) - build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_ADD, lhs, - ops[0], ops[1]); + return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_ADD, + lhs, ops[0], ops[1]); + + return false; } /* @@ -4257,17 +4263,20 @@ match_saturation_add (gimple_stmt_iterator *gsi, gphi *phi) * _3 = _4 - _5; * _6 = _1 ? _3 : 0; * => - * _6 = .SAT_SUB (_4, _5); */ + * _6 = .SAT_SUB (_4, _5); + * Return true if the statement was replaced. */ -static void +static bool match_unsigned_saturation_sub (gimple_stmt_iterator *gsi, gassign *stmt) { tree ops[2]; tree lhs = gimple_assign_lhs (stmt); if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL)) - build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_SUB, lhs, - ops[0], ops[1]); + return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_SUB, + lhs, ops[0], ops[1]); + + return false; } /* @@ -7335,11 +7344,14 @@ math_opts_dom_walker::after_dom_children (basic_block bb) break; case PLUS_EXPR: - match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt)); - match_unsigned_saturation_sub (&gsi, as_a<gassign *> (stmt)); + if (match_saturation_add_with_assign (&gsi, + as_a<gassign *> (stmt))) + break; /* fall-through */ case MINUS_EXPR: - if (!convert_plusminus_to_widen (&gsi, stmt, code)) + if (!match_unsigned_saturation_sub (&gsi, + as_a<gassign *> (stmt)) + && !convert_plusminus_to_widen (&gsi, stmt, code)) { match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p); if (gsi_stmt (gsi) == stmt)
