From: Kyrylo Tkachov <[email protected]>
A signed value clamped to the range of a narrower unsigned type is unsigned
saturating truncation. Recognize nested MIN and MAX operations in either
order.
Keep the clamp match independent of use count. At this use, require an exact
zero lower bound, the maximum upper bound, and one use of the outer clamp.
The inner clamp can have other uses.
With AArch64 -O2 -march=armv8-a -fopenmp-simd:
before:
clamp_u16_4:
ldr q0, [x1]
movi v30.4s, 0xff, msl 8
movi v31.4s, 0
smin v30.4s, v0.4s, v30.4s
smax v30.4s, v30.4s, v31.4s
xtn v30.4h, v30.4s
str d30, [x0]
ret
after:
clamp_u16_4:
ldr q31, [x1]
movi v30.4s, 0
smax v30.4s, v31.4s, v30.4s
uqxtn v30.4h, v30.4s
str d30, [x0]
ret
Runtime tests cover both clamp orders, two widths, and LTO. Target tests
cover valid bounds, invalid bounds, and extra uses of each clamp.
Bootstrapped and tested on aarch64-none-linux-gnu.
Ok for trunk?
Changes since v3:
- Make the clamp match independent of use count.
- Check the outer clamp use at the consumer.
- Match the zero bound with integer_zerop.
gcc/ChangeLog:
* match-sat-alu.pd (integer_constant_clamp): New match.
(unsigned_integer_narrow_clip): Add the constant-clamp form.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-sat-trunc-clamp-1.c: New test.
* gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c: Likewise.
* gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c: Likewise.
* gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c: Likewise.
* gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c: Likewise.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
gcc/match-sat-alu.pd | 23 +++-
.../gcc.dg/vect/vect-sat-trunc-clamp-1.c | 121 ++++++++++++++++++
.../vect-sat-trunc-clamp-inner-use-1.c | 22 ++++
.../vect-sat-trunc-clamp-multi-use-1.c | 21 +++
.../aarch64/vect-sat-trunc-clamp-u-1.c | 7 +
.../vect-sat-trunc-clamp-u-negative-1.c | 30 +++++
6 files changed, 223 insertions(+), 1 deletion(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c
create mode 100644
gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c
diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd
index 5a5ed52d40d..c4b24dfab11 100644
--- a/gcc/match-sat-alu.pd
+++ b/gcc/match-sat-alu.pd
@@ -23,6 +23,13 @@ along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
+/* Clamp @0 between the constant bounds @1 and @2. The source can put the
+ MIN or the MAX first. */
+(match (integer_constant_clamp @0 @1 @2)
+ (max (min @0 INTEGER_CST@2) INTEGER_CST@1))
+(match (integer_constant_clamp @0 @1 @2)
+ (min (max @0 INTEGER_CST@1) INTEGER_CST@2))
+
/* Saturation add for unsigned integer. */
(if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type))
(match (usadd_overflow_mask @0 @1)
@@ -293,7 +300,21 @@ along with GCC; see the file COPYING3. If not see
&& wi::eq_p (trunc_max, int_cst_1)
&& cmp >= 0
&& expr_not_equal_to (@0, wi::min_value (itype_precision,
- SIGNED))))))))
+ SIGNED)))))))
+ (match (unsigned_integer_narrow_clip @0)
+ /* SAT_US_TRUNC = (NT)MAX (MIN (X, NT_MAX), 0). */
+ (convert (integer_constant_clamp@3 @0 integer_zerop@1 @2))
+ (if (!TYPE_UNSIGNED (TREE_TYPE (@0))
+ && single_use (@3))
+ (with
+ {
+ unsigned itype_precision = TYPE_PRECISION (TREE_TYPE (@0));
+ unsigned otype_precision = TYPE_PRECISION (type);
+ wide_int trunc_max = wi::mask (otype_precision, false, itype_precision);
+ wide_int hi_cst = wi::to_wide (@2, itype_precision);
+ }
+ (if (otype_precision < itype_precision
+ && wi::eq_p (hi_cst, trunc_max)))))))
/* Saturation truncate for unsigned integer. */
(if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type))
diff --git a/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c
b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c
new file mode 100644
index 00000000000..06a4428d0d2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c
@@ -0,0 +1,121 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fvect-cost-model=unlimited" } */
+
+#include "tree-vect.h"
+
+typedef __UINT16_TYPE__ u16;
+typedef __UINT32_TYPE__ u32;
+typedef __UINT64_TYPE__ u64;
+typedef __INT32_TYPE__ i32;
+typedef __INT64_TYPE__ i64;
+
+#define N 259
+
+static i32 in32[N];
+static i64 in64[N];
+static u16 out16[N];
+static u32 out32[N];
+
+__attribute__((noipa)) static void
+clamp_u16_lo (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i32 x = in[i];
+ out[i] = x < 0 ? 0 : (x > 65535 ? 65535 : x);
+ }
+}
+
+__attribute__((noipa)) static void
+clamp_u16_hi (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i32 x = in[i];
+ out[i] = x > 65535 ? 65535 : (x < 0 ? 0 : x);
+ }
+}
+
+__attribute__((noipa)) static void
+clamp_u32_lo (u32 *__restrict out, const i64 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i64 x = in[i];
+ out[i] = x < 0 ? 0 : (x > 4294967295LL ? 4294967295LL : x);
+ }
+}
+
+__attribute__((noipa)) static void
+clamp_u32_hi (u32 *__restrict out, const i64 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i64 x = in[i];
+ out[i] = x > 4294967295LL ? 4294967295LL : (x < 0 ? 0 : x);
+ }
+}
+
+static u16
+ref_u16 (i32 x)
+{
+ return x < 0 ? 0 : (x > 65535 ? 65535 : x);
+}
+
+static u32
+ref_u32 (i64 x)
+{
+ return x < 0 ? 0 : (x > 4294967295LL ? 4294967295LL : x);
+}
+
+static void
+check_u16 (void)
+{
+#pragma GCC novector
+ for (int i = 0; i < N; ++i)
+ if (out16[i] != ref_u16 (in32[i]))
+ __builtin_abort ();
+}
+
+static void
+check_u32 (void)
+{
+#pragma GCC novector
+ for (int i = 0; i < N; ++i)
+ if (out32[i] != ref_u32 (in64[i]))
+ __builtin_abort ();
+}
+
+int
+main (void)
+{
+ check_vect ();
+
+ for (int i = 0; i < N; ++i)
+ {
+ in32[i] = (i32) ((u32) i * 2654435761U + 1013904223U);
+ in64[i] = (i64) ((u64) i * 11400714819323198485ULL
+ + 13787848793156543929ULL);
+ }
+
+ in32[0] = -1;
+ in32[1] = 0;
+ in32[2] = 65535;
+ in32[3] = 65536;
+ in64[0] = -1;
+ in64[1] = 0;
+ in64[2] = 4294967295LL;
+ in64[3] = 4294967296LL;
+
+ clamp_u16_lo (out16, in32, N);
+ check_u16 ();
+ clamp_u16_hi (out16, in32, N);
+ check_u16 ();
+ clamp_u32_lo (out32, in64, N);
+ check_u32 ();
+ clamp_u32_hi (out32, in64, N);
+ check_u32 ();
+
+ return 0;
+}
diff --git
a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c
b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c
new file mode 100644
index 00000000000..b93bc56188e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization
--param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __INT32_TYPE__ i32;
+
+void
+clip (u16 *__restrict out, i32 *__restrict copy,
+ const i32 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i32 x = in[i];
+ i32 inner = x > 65535 ? 65535 : x;
+ i32 outer = inner < 0 ? 0 : inner;
+ out[i] = outer;
+ copy[i] = inner;
+ }
+}
+
+/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 1 "vect" }
} */
diff --git
a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c
b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c
new file mode 100644
index 00000000000..0b82d40ff2f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization
--param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __INT32_TYPE__ i32;
+
+void
+clip (u16 *__restrict out, i32 *__restrict copy,
+ const i32 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i32 x = in[i];
+ i32 clamped = x < 0 ? 0 : (x > 65535 ? 65535 : x);
+ out[i] = clamped;
+ copy[i] = clamped;
+ }
+}
+
+/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c
b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c
new file mode 100644
index 00000000000..f159a8f4dac
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c
@@ -0,0 +1,7 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization
--param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+#include "../../gcc.dg/vect/vect-sat-trunc-clamp-1.c"
+
+/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 4 "vect" }
} */
diff --git
a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c
b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c
new file mode 100644
index 00000000000..aa283f176ad
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization
--param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __INT32_TYPE__ i32;
+
+__attribute__((noipa))
+void
+bad_low (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i32 x = in[i];
+ out[i] = x < 1 ? 1 : (x > 65535 ? 65535 : x);
+ }
+}
+
+__attribute__((noipa))
+void
+bad_high (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ i32 x = in[i];
+ out[i] = x > 65534 ? 65534 : (x < 0 ? 0 : x);
+ }
+}
+
+/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */
--
2.50.1 (Apple Git-155)
