From: Kyrylo Tkachov <[email protected]>
X - MIN (X, Y) is unsigned saturating subtraction, but subtraction
statements were not checked for this form.
Recognize the form and check MINUS_EXPR statements for saturation. Require
the MIN to have one use so that the replacement removes it. Make the assign
recognizers report whether they replaced the statement, and do not run later
transforms after a replacement.
AArch64 -O2:
before:
cmp w1, w0
csel w1, w1, w0, ls
sub w0, w0, w1
after:
subs w0, w0, w1
csel w0, w0, wzr, cs
The vector form becomes one UQSUB instead of a comparison, a select, and a
subtraction.
Bootstrapped and tested on aarch64-none-linux-gnu.
Ok for trunk?
Changes since v2:
- Rebase on current trunk.
gcc/ChangeLog:
* match-sat-alu.pd (unsigned_integer_sat_sub): Add the
X - MIN (X, Y) form.
* tree-ssa-math-opts.cc
(build_saturation_binary_arith_call_and_replace): Return whether a
replacement was made.
(match_saturation_add_with_assign): Likewise.
(match_unsigned_saturation_sub): Likewise.
(math_opts_dom_walker::after_dom_children): Check MINUS_EXPR for
saturation, and use the assign recognizer results.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sat_u_sub_minmax-1.c: New test.
* gcc.target/aarch64/sat_u_sub_minmax-2.c: New test.
* gcc.target/aarch64/sat_u_sub_minmax-3.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/match-sat-alu.pd | 6 ++++
.../gcc.target/aarch64/sat_u_sub_minmax-1.c | 18 ++++++++++
.../gcc.target/aarch64/sat_u_sub_minmax-2.c | 32 +++++++++++++++++
.../gcc.target/aarch64/sat_u_sub_minmax-3.c | 17 +++++++++
gcc/tree-ssa-math-opts.cc | 36 ++++++++++++-------
5 files changed, 97 insertions(+), 12 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-3.c
diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd
index 2a37d23a869..f7ee8d63f1e 100644
--- a/gcc/match-sat-alu.pd
+++ b/gcc/match-sat-alu.pd
@@ -169,6 +169,12 @@ along with GCC; see the file COPYING3. If not see
(match (unsigned_integer_sat_sub @0 @1)
/* SAT_U_SUB = (X - Y) <= X ? X - Y : 0, and the commuted comparison. */
(cond^ (le:c (minus@2 @0 @1) @0) @2 integer_zerop))
+ (match (unsigned_integer_sat_sub @0 @1)
+ /* SAT_U_SUB = X - MIN (X, Y). The MIN has to be single use: while it
+ stays live the saturating subtract is computed beside it instead of
+ replacing it, which costs an instruction. */
+ (minus @0 (min:c@2 @0 @1))
+ (if (single_use (@2))))
(match (unsigned_integer_sat_sub @0 @1)
/* DIFF = SUB_OVERFLOW (X, Y)
SAT_U_SUB = REALPART (DIFF) | (IMAGPART (DIFF) + (-1)) */
diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-1.c
b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-1.c
new file mode 100644
index 00000000000..aa1d7b6f031
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-1.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+#define DEF_MIN(T) \
+ T min_##T (T a, T b) { return a - (a < b ? a : b); } \
+ T nim_##T (T a, T b) { return a - (b < a ? b : a); }
+
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long long u64;
+
+DEF_MIN (u8)
+DEF_MIN (u16)
+DEF_MIN (u32)
+DEF_MIN (u64)
+
+/* { dg-final { scan-tree-dump-times "\\.SAT_SUB " 8 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-2.c
b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-2.c
new file mode 100644
index 00000000000..dcf8ab46309
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-2.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-optimized" } */
+
+typedef unsigned char u8;
+typedef unsigned int u32;
+
+/* The recognition must reach the vectoriser, so that the loop becomes a
+ single uqsub rather than a compare, a select and a subtraction. */
+
+void
+min_loop (u8 *__restrict d, u8 *__restrict a, u8 *__restrict b, int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ u8 x = a[i], y = b[i];
+ d[i] = x - (x < y ? x : y);
+ }
+}
+
+void
+min_loop32 (u32 *__restrict d, u32 *__restrict a, u32 *__restrict b, int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ u32 x = a[i], y = b[i];
+ d[i] = x - (y < x ? y : x);
+ }
+}
+
+/* { dg-final { scan-tree-dump "\\.SAT_SUB " "optimized" } } */
+/* { dg-final { scan-assembler "uqsub\tv\[0-9\]+\\.16b" } } */
+/* { dg-final { scan-assembler "uqsub\tv\[0-9\]+\\.4s" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-3.c
b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-3.c
new file mode 100644
index 00000000000..ca71343699d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sat_u_sub_minmax-3.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+typedef unsigned int u32;
+
+/* While the MIN stays live the saturating subtract would be computed beside
+ it rather than instead of it, so the rules do not fire. */
+
+u32
+min_live (u32 a, u32 b, u32 *o)
+{
+ u32 m = a < b ? a : b;
+ *o = m;
+ return a - m;
+}
+
+/* { dg-final { scan-tree-dump-not "\\.SAT_SUB " "optimized" } } */
diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc
index 70ac17c9e2f..9c959cc8450 100644
--- a/gcc/tree-ssa-math-opts.cc
+++ b/gcc/tree-ssa-math-opts.cc
@@ -4123,7 +4123,7 @@ extern bool gimple_signed_integer_sat_add (tree, tree*,
tree (*)(tree));
extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
-static void
+static bool
build_saturation_binary_arith_call_and_replace (gimple_stmt_iterator *gsi,
internal_fn fn, tree lhs,
tree op_0, tree op_1)
@@ -4133,7 +4133,10 @@ build_saturation_binary_arith_call_and_replace
(gimple_stmt_iterator *gsi,
gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
gimple_call_set_lhs (call, lhs);
gsi_replace (gsi, call, /* update_eh_info */ true);
+ return true;
}
+
+ return false;
}
static bool
@@ -4177,9 +4180,10 @@ build_saturation_binary_arith_call_and_insert
(gimple_stmt_iterator *gsi,
* # _9 = PHI <_2(3), 128(2)>
* _4 = (int8_t) _9;
* =>
- * _4 = .SAT_ADD (x_5, -1); */
+ * _4 = .SAT_ADD (x_5, -1);
+ * Return true if the statement was replaced. */
-static void
+static bool
match_saturation_add_with_assign (gimple_stmt_iterator *gsi, gassign *stmt)
{
tree ops[2];
@@ -4187,8 +4191,10 @@ match_saturation_add_with_assign (gimple_stmt_iterator
*gsi, gassign *stmt)
if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
|| gimple_signed_integer_sat_add (lhs, ops, NULL))
- build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_ADD, lhs,
- ops[0], ops[1]);
+ return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_ADD,
+ lhs, ops[0], ops[1]);
+
+ return false;
}
/*
@@ -4257,17 +4263,20 @@ match_saturation_add (gimple_stmt_iterator *gsi, gphi
*phi)
* _3 = _4 - _5;
* _6 = _1 ? _3 : 0;
* =>
- * _6 = .SAT_SUB (_4, _5); */
+ * _6 = .SAT_SUB (_4, _5);
+ * Return true if the statement was replaced. */
-static void
+static bool
match_unsigned_saturation_sub (gimple_stmt_iterator *gsi, gassign *stmt)
{
tree ops[2];
tree lhs = gimple_assign_lhs (stmt);
if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL))
- build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_SUB, lhs,
- ops[0], ops[1]);
+ return build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_SUB,
+ lhs, ops[0], ops[1]);
+
+ return false;
}
/*
@@ -7335,11 +7344,14 @@ math_opts_dom_walker::after_dom_children (basic_block
bb)
break;
case PLUS_EXPR:
- match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
- match_unsigned_saturation_sub (&gsi, as_a<gassign *> (stmt));
+ if (match_saturation_add_with_assign (&gsi,
+ as_a<gassign *> (stmt)))
+ break;
/* fall-through */
case MINUS_EXPR:
- if (!convert_plusminus_to_widen (&gsi, stmt, code))
+ if (!match_unsigned_saturation_sub (&gsi,
+ as_a<gassign *> (stmt))
+ && !convert_plusminus_to_widen (&gsi, stmt, code))
{
match_arith_overflow (&gsi, stmt, code, m_cfg_changed_p);
if (gsi_stmt (gsi) == stmt)
--
2.50.1 (Apple Git-155)
