From: Kyrylo Tkachov <[email protected]>
For an unsigned type, MAX - Y is ~Y. An overflow test can therefore use
X > ~Y ? MAX : X + Y
Recognize strict and non-strict comparisons with either operand or result
arm order. Equality is safe because both arms produce MAX. Also recognize
ADD_OVERFLOW forms that select a separate X + Y expression.
The comparison iterators cover strict and non-strict forms. The `:c`
modifier covers both operand orders.
AArch64 -O3, vector body:
before:
add v29.4s, v31.4s, v30.4s
not v30.16b, v30.16b
cmhs v30.4s, v30.4s, v31.4s
orn v30.16b, v29.16b, v30.16b
after:
uqadd v30.4s, v31.4s, v30.4s
Four vector operations become one.
Bootstrapped and regression tested on aarch64-unknown-linux-gnu.
Ok for trunk?
Changes since v3:
- Rebase on current trunk and drop the overflow canonicalization
prerequisite, which is now upstream.
- Use the general genmatch handling for overlapping operator sets.
gcc/ChangeLog:
* match-sat-alu.pd (unsigned_integer_sat_add): Recognise comparisons
against the complement of an operand. Also recognise ADD_OVERFLOW
with a separate addition.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sat_u_add_not_cmp-1.c: New test.
* gcc.target/aarch64/sat_u_add_not_cmp-2.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/match-sat-alu.pd | 22 ++++++++++++++
.../gcc.target/aarch64/sat_u_add_not_cmp-1.c | 30 +++++++++++++++++++
.../gcc.target/aarch64/sat_u_add_not_cmp-2.c | 16 ++++++++++
3 files changed, 68 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-2.c
diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd
index 83cd8963147..e787a7414f4 100644
--- a/gcc/match-sat-alu.pd
+++ b/gcc/match-sat-alu.pd
@@ -104,6 +104,28 @@ along with GCC; see the file COPYING3. If not see
(cond^ (ne (imagpart (IFN_ADD_OVERFLOW@2 @0 INTEGER_CST@1)) integer_zerop)
integer_minus_onep (realpart @2))
(if (types_match (type, @0) && int_fits_type_p (@1, type))))
+ (match (unsigned_integer_sat_add @0 @1)
+ /* SUM = ADD_OVERFLOW (X, Y)
+ SAT_U_ADD = IMAGPART (SUM) == 0 ? X + Y : -1. */
+ (cond^ (eq (imagpart (IFN_ADD_OVERFLOW @0 @1)) integer_zerop)
+ (plus @0 @1) integer_minus_onep))
+ (match (unsigned_integer_sat_add @0 @1)
+ /* SUM = ADD_OVERFLOW (X, Y)
+ SAT_U_ADD = IMAGPART (SUM) != 0 ? -1 : X + Y. */
+ (cond^ (ne (imagpart (IFN_ADD_OVERFLOW @0 @1)) integer_zerop)
+ integer_minus_onep (plus @0 @1)))
+ /* A source that tests for overflow against the complement of an operand,
+ such as a > MAX - b ? MAX : a + b, reaches the recognisers as a compare
+ with ~Y, since ~Y is MAX - Y. The strict and non-strict forms are both
+ exact because at X == ~Y the sum is MAX and the two arms agree. */
+ (for cmp (ge gt)
+ (match (unsigned_integer_sat_add @0 @1)
+ (cond^ (cmp:c (bit_not @1) @0)
+ (plus:c @0 @1) integer_minus_onep)))
+ (for cmp (lt le)
+ (match (unsigned_integer_sat_add @0 @1)
+ (cond^ (cmp:c (bit_not @1) @0)
+ integer_minus_onep (plus:c @0 @1))))
(match (unsigned_integer_sat_add @0 @1)
/* WIDEN_SUM = (WT)X + (WT)Y
SAT_U_ADD = WIDEN_SUM > MAX ? MAX : (NT)WIDEN_SUM */
diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-1.c
b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-1.c
new file mode 100644
index 00000000000..88041f53f61
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+typedef unsigned long long u64;
+
+/* Every one of these tests for overflow against MAX - b, which reaches the
+ middle end as a comparison with ~b. The strict and the non-strict form
+ agree, because at a == MAX - b the sum is MAX and so is the other arm. */
+
+#define DEF(N, T, MX) \
+ T f1_##N (T a, T b) { return a > (T) (MX - b) ? MX : a + b; } \
+ T f2_##N (T a, T b) { return a >= (T) (MX - b) ? MX : a + b; } \
+ T f3_##N (T a, T b) { return b > (T) ~a ? MX : a + b; } \
+ T f4_##N (T a, T b) { return b >= (T) ~a ? MX : a + b; } \
+ T f5_##N (T a, T b) { return a <= (T) (MX - b) ? a + b : MX; } \
+ T f6_##N (T a, T b) { return a < (T) (MX - b) ? a + b : MX; } \
+ T f7_##N (T a, T b) { return (T) (MX - a) < b ? MX : a + b; } \
+ T f8_##N (T a, T b) { return (T) (MX - a) <= b ? MX : a + b; } \
+ T f9_##N (T a, T b) { return (T) ~b >= a ? a + b : MX; } \
+ T f10_##N (T a, T b) { return (T) ~b > a ? a + b : MX; }
+
+DEF (8, u8, 255)
+DEF (16, u16, 65535)
+DEF (32, u32, 0xffffffffu)
+DEF (64, u64, ~0ull)
+
+/* { dg-final { scan-tree-dump-times "\\.SAT_ADD " 40 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-2.c
b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-2.c
new file mode 100644
index 00000000000..bcd81304f91
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sat_u_add_not_cmp-2.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-optimized" } */
+
+typedef unsigned int u32;
+
+/* Recognition lets the loop vectorize to one uqadd. */
+
+void
+f (u32 *__restrict d, u32 *__restrict a, u32 *__restrict b, int n)
+{
+ for (int i = 0; i < n; i++)
+ d[i] = a[i] > 0xffffffffu - b[i] ? 0xffffffffu : a[i] + b[i];
+}
+
+/* { dg-final { scan-tree-dump "\\.SAT_ADD " "optimized" } } */
+/* { dg-final { scan-assembler "uqadd\tv\[0-9\]+\.4s" } } */
--
2.50.1 (Apple Git-155)
