From: Kyrylo Tkachov <[email protected]>
vect_recog_popcount_clz_ctz_ffs_pattern checks the second argument of a
two-argument .CLZ or .CTZ against the value of the target. It does
this only when the precisions of the argument and the result differ.
For equal precisions it builds the pattern call with the value of the
target. On AArch64
_5 = .CLZ (_1, 5);
_9 = (unsigned int) _5;
becomes
vect_patt_18.11_23 = .CLZ (vect__1.10_22, 32);
and zero lanes give 32 instead of 5. The expansion implements any value
at zero of a scalar call, so such scalar calls are valid.
Do the check after the switch, for all precisions. The calls of the
example then stay scalar.
Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux.
gcc/ChangeLog:
* tree-vect-patterns.cc (vect_recog_popcount_clz_ctz_ffs_pattern):
Check the value at zero of a two-argument .CLZ or .CTZ for all
precisions.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c: New test.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
.../vect/bb-slp-clz-ctz-value-at-zero-1.c | 182 ++++++++++++++++++
gcc/tree-vect-patterns.cc | 43 ++---
2 files changed, 199 insertions(+), 26 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
new file mode 100644
index 00000000000..64404dc01e0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
@@ -0,0 +1,182 @@
+/* Lanes of .CLZ and .CTZ whose value at zero is not the value of the
+ target must keep that value. */
+/* { dg-require-effective-target int32 } */
+/* { dg-require-effective-target clz_ifn } */
+/* { dg-require-effective-target ctz_ifn } */
+/* { dg-additional-options "-fgimple -fdump-tree-optimized" } */
+
+#include "tree-vect.h"
+
+unsigned int a[4];
+int r[4];
+unsigned int u[4];
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+clz5 (void)
+{
+ unsigned int _1;
+ unsigned int _2;
+ unsigned int _3;
+ unsigned int _4;
+ int _5;
+ int _6;
+ int _7;
+ int _8;
+
+ __BB(2):
+ _1 = a[0];
+ _2 = a[1];
+ _3 = a[2];
+ _4 = a[3];
+ _5 = .CLZ (_1, 5);
+ _6 = .CLZ (_2, 5);
+ _7 = .CLZ (_3, 5);
+ _8 = .CLZ (_4, 5);
+ r[0] = _5;
+ r[1] = _6;
+ r[2] = _7;
+ r[3] = _8;
+ return;
+}
+
+/* The conversion makes the call part of a vectorizer pattern. */
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+clz5u (void)
+{
+ unsigned int _1;
+ unsigned int _2;
+ unsigned int _3;
+ unsigned int _4;
+ int _5;
+ int _6;
+ int _7;
+ int _8;
+ unsigned int _9;
+ unsigned int _10;
+ unsigned int _11;
+ unsigned int _12;
+
+ __BB(2):
+ _1 = a[0];
+ _2 = a[1];
+ _3 = a[2];
+ _4 = a[3];
+ _5 = .CLZ (_1, 5);
+ _6 = .CLZ (_2, 5);
+ _7 = .CLZ (_3, 5);
+ _8 = .CLZ (_4, 5);
+ _9 = (unsigned int) _5;
+ _10 = (unsigned int) _6;
+ _11 = (unsigned int) _7;
+ _12 = (unsigned int) _8;
+ u[0] = _9;
+ u[1] = _10;
+ u[2] = _11;
+ u[3] = _12;
+ return;
+}
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+ctz5u (void)
+{
+ unsigned int _1;
+ unsigned int _2;
+ unsigned int _3;
+ unsigned int _4;
+ int _5;
+ int _6;
+ int _7;
+ int _8;
+ unsigned int _9;
+ unsigned int _10;
+ unsigned int _11;
+ unsigned int _12;
+
+ __BB(2):
+ _1 = a[0];
+ _2 = a[1];
+ _3 = a[2];
+ _4 = a[3];
+ _5 = .CTZ (_1, 5);
+ _6 = .CTZ (_2, 5);
+ _7 = .CTZ (_3, 5);
+ _8 = .CTZ (_4, 5);
+ _9 = (unsigned int) _5;
+ _10 = (unsigned int) _6;
+ _11 = (unsigned int) _7;
+ _12 = (unsigned int) _8;
+ u[0] = _9;
+ u[1] = _10;
+ u[2] = _11;
+ u[3] = _12;
+ return;
+}
+
+/* 32 is the value of the target on AArch64, so this vectorizes there. */
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+clz32u (void)
+{
+ unsigned int _1;
+ unsigned int _2;
+ unsigned int _3;
+ unsigned int _4;
+ int _5;
+ int _6;
+ int _7;
+ int _8;
+ unsigned int _9;
+ unsigned int _10;
+ unsigned int _11;
+ unsigned int _12;
+
+ __BB(2):
+ _1 = a[0];
+ _2 = a[1];
+ _3 = a[2];
+ _4 = a[3];
+ _5 = .CLZ (_1, 32);
+ _6 = .CLZ (_2, 32);
+ _7 = .CLZ (_3, 32);
+ _8 = .CLZ (_4, 32);
+ _9 = (unsigned int) _5;
+ _10 = (unsigned int) _6;
+ _11 = (unsigned int) _7;
+ _12 = (unsigned int) _8;
+ u[0] = _9;
+ u[1] = _10;
+ u[2] = _11;
+ u[3] = _12;
+ return;
+}
+
+static void __attribute__((noipa))
+check (const unsigned int *p, unsigned int e0, unsigned int e1,
+ unsigned int e2, unsigned int e3)
+{
+ if (p[0] != e0 || p[1] != e1 || p[2] != e2 || p[3] != e3)
+ __builtin_abort ();
+}
+
+int
+main (void)
+{
+ check_vect ();
+
+ /* Write A at run time, so that LTO cannot fold the loads. */
+ a[1] = 1;
+ a[3] = 0x80000000u;
+ clz5 ();
+ check ((const unsigned int *) r, 5, 31, 5, 0);
+ clz5u ();
+ check (u, 5, 31, 5, 0);
+ ctz5u ();
+ check (u, 5, 0, 5, 31);
+ clz32u ();
+ check (u, 32, 31, 32, 0);
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 1 "optimized" { target
aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-not {\.CTZ \(vect_} "optimized" { target
aarch64*-*-* } } } */
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index e9d2870feb4..7c435adeb98 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -2260,9 +2260,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
return NULL;
break;
case IFN_CLZ:
- /* clzll (x) == clz (x) + 32 for unsigned x != 0, so ok
- if it is undefined at zero or if it matches also for the
- defined value there. */
+ /* clzll (x) == clz (x) + 32 for unsigned x != 0. */
if (!TYPE_UNSIGNED (unprom_diff.type))
return NULL;
if (!type_has_mode_precision_p (lhs_type)
@@ -2270,31 +2268,9 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
return NULL;
addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
- TYPE_PRECISION (lhs_type));
- if (gimple_call_internal_p (call_stmt)
- && gimple_call_num_args (call_stmt) == 2)
- {
- int val1, val2;
- val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
- int d2
- = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
- if (d2 != 2 || val1 != val2 + addend)
- return NULL;
- }
break;
case IFN_CTZ:
- /* ctzll (x) == ctz (x) for unsigned or signed x != 0, so ok
- if it is undefined at zero or if it matches also for the
- defined value there. */
- if (gimple_call_internal_p (call_stmt)
- && gimple_call_num_args (call_stmt) == 2)
- {
- int val1, val2;
- val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
- int d2
- = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
- if (d2 != 2 || val1 != val2)
- return NULL;
- }
+ /* ctzll (x) == ctz (x) for unsigned or signed x != 0. */
break;
case IFN_FFS:
/* ffsll (x) == ffs (x) for unsigned or signed x. */
@@ -2303,6 +2279,21 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
gcc_unreachable ();
}
+ /* For zero the pattern gives the value of the target for VEC_TYPE plus
+ ADDEND. Punt if the call has another value at zero. */
+ if ((ifn == IFN_CLZ || ifn == IFN_CTZ)
+ && gimple_call_internal_p (call_stmt)
+ && gimple_call_num_args (call_stmt) == 2)
+ {
+ int val;
+ int d = (ifn == IFN_CLZ
+ ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val)
+ : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val));
+ if (d != 2
+ || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend)
+ return NULL;
+ }
+
/* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
bool supported
= direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
--
2.54.0 (Apple Git-157)
