https://gcc.gnu.org/g:3b272406eddd7f82749d67483022c7198cdbfc3e
commit r17-5897-g3b272406eddd7f82749d67483022c7198cdbfc3e Author: Kyrylo Tkachov <[email protected]> Date: Fri Oct 9 08:53:43 2026 -0700 vect: Check the C[LT]Z value at zero for all precisions vect_recog_popcount_clz_ctz_ffs_pattern checks the second argument of a two-argument .CLZ or .CTZ against the value of the target. It does this only when the precisions of the argument and the result differ. For equal precisions it builds the pattern call with the value of the target. On AArch64 _5 = .CLZ (_1, 5); _9 = (unsigned int) _5; becomes vect_patt_18.11_23 = .CLZ (vect__1.10_22, 32); and zero lanes give 32 instead of 5. The expansion implements any value at zero of a scalar call, so such scalar calls are valid. Do the check after the switch, for all precisions. The calls of the example then stay scalar. Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux. gcc/ChangeLog: * tree-vect-patterns.cc (vect_recog_popcount_clz_ctz_ffs_pattern): Check the value at zero of a two-argument .CLZ or .CTZ for all precisions. gcc/testsuite/ChangeLog: * gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c: New test. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- .../gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c | 182 +++++++++++++++++++++ gcc/tree-vect-patterns.cc | 43 ++--- 2 files changed, 199 insertions(+), 26 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c new file mode 100644 index 000000000000..23b5c428bbe2 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c @@ -0,0 +1,182 @@ +/* Lanes of .CLZ and .CTZ whose value at zero is not the value of the + target must keep that value. */ +/* { dg-require-effective-target int32 } */ +/* { dg-require-effective-target clzsi } */ +/* { dg-require-effective-target ctzsi } */ +/* { dg-additional-options "-fgimple -fdump-tree-optimized" } */ + +#include "tree-vect.h" + +unsigned int a[4]; +int r[4]; +unsigned int u[4]; + +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa)) +clz5 (void) +{ + unsigned int _1; + unsigned int _2; + unsigned int _3; + unsigned int _4; + int _5; + int _6; + int _7; + int _8; + + __BB(2): + _1 = a[0]; + _2 = a[1]; + _3 = a[2]; + _4 = a[3]; + _5 = .CLZ (_1, 5); + _6 = .CLZ (_2, 5); + _7 = .CLZ (_3, 5); + _8 = .CLZ (_4, 5); + r[0] = _5; + r[1] = _6; + r[2] = _7; + r[3] = _8; + return; +} + +/* The conversion makes the call part of a vectorizer pattern. */ + +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa)) +clz5u (void) +{ + unsigned int _1; + unsigned int _2; + unsigned int _3; + unsigned int _4; + int _5; + int _6; + int _7; + int _8; + unsigned int _9; + unsigned int _10; + unsigned int _11; + unsigned int _12; + + __BB(2): + _1 = a[0]; + _2 = a[1]; + _3 = a[2]; + _4 = a[3]; + _5 = .CLZ (_1, 5); + _6 = .CLZ (_2, 5); + _7 = .CLZ (_3, 5); + _8 = .CLZ (_4, 5); + _9 = (unsigned int) _5; + _10 = (unsigned int) _6; + _11 = (unsigned int) _7; + _12 = (unsigned int) _8; + u[0] = _9; + u[1] = _10; + u[2] = _11; + u[3] = _12; + return; +} + +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa)) +ctz5u (void) +{ + unsigned int _1; + unsigned int _2; + unsigned int _3; + unsigned int _4; + int _5; + int _6; + int _7; + int _8; + unsigned int _9; + unsigned int _10; + unsigned int _11; + unsigned int _12; + + __BB(2): + _1 = a[0]; + _2 = a[1]; + _3 = a[2]; + _4 = a[3]; + _5 = .CTZ (_1, 5); + _6 = .CTZ (_2, 5); + _7 = .CTZ (_3, 5); + _8 = .CTZ (_4, 5); + _9 = (unsigned int) _5; + _10 = (unsigned int) _6; + _11 = (unsigned int) _7; + _12 = (unsigned int) _8; + u[0] = _9; + u[1] = _10; + u[2] = _11; + u[3] = _12; + return; +} + +/* 32 is the value of the target on AArch64, so this vectorizes there. */ + +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa)) +clz32u (void) +{ + unsigned int _1; + unsigned int _2; + unsigned int _3; + unsigned int _4; + int _5; + int _6; + int _7; + int _8; + unsigned int _9; + unsigned int _10; + unsigned int _11; + unsigned int _12; + + __BB(2): + _1 = a[0]; + _2 = a[1]; + _3 = a[2]; + _4 = a[3]; + _5 = .CLZ (_1, 32); + _6 = .CLZ (_2, 32); + _7 = .CLZ (_3, 32); + _8 = .CLZ (_4, 32); + _9 = (unsigned int) _5; + _10 = (unsigned int) _6; + _11 = (unsigned int) _7; + _12 = (unsigned int) _8; + u[0] = _9; + u[1] = _10; + u[2] = _11; + u[3] = _12; + return; +} + +static void __attribute__((noipa)) +check (const unsigned int *p, unsigned int e0, unsigned int e1, + unsigned int e2, unsigned int e3) +{ + if (p[0] != e0 || p[1] != e1 || p[2] != e2 || p[3] != e3) + __builtin_abort (); +} + +int +main (void) +{ + check_vect (); + + /* Write A at run time, so that LTO cannot fold the loads. */ + a[1] = 1; + a[3] = 0x80000000u; + clz5 (); + check ((const unsigned int *) r, 5, 31, 5, 0); + clz5u (); + check (u, 5, 31, 5, 0); + ctz5u (); + check (u, 5, 0, 5, 31); + clz32u (); + check (u, 32, 31, 32, 0); + return 0; +} + +/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 1 "optimized" { target aarch64*-*-* } } } */ +/* { dg-final { scan-tree-dump-not {\.CTZ \(vect_} "optimized" { target aarch64*-*-* } } } */ diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc index d79f625280a6..c706dafd6111 100644 --- a/gcc/tree-vect-patterns.cc +++ b/gcc/tree-vect-patterns.cc @@ -2258,9 +2258,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, return NULL; break; case IFN_CLZ: - /* clzll (x) == clz (x) + 32 for unsigned x != 0, so ok - if it is undefined at zero or if it matches also for the - defined value there. */ + /* clzll (x) == clz (x) + 32 for unsigned x != 0. */ if (!TYPE_UNSIGNED (unprom_diff.type)) return NULL; if (!type_has_mode_precision_p (lhs_type) @@ -2268,31 +2266,9 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, return NULL; addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd)) - TYPE_PRECISION (lhs_type)); - if (gimple_call_internal_p (call_stmt) - && gimple_call_num_args (call_stmt) == 2) - { - int val1, val2; - val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1)); - int d2 - = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2); - if (d2 != 2 || val1 != val2 + addend) - return NULL; - } break; case IFN_CTZ: - /* ctzll (x) == ctz (x) for unsigned or signed x != 0, so ok - if it is undefined at zero or if it matches also for the - defined value there. */ - if (gimple_call_internal_p (call_stmt) - && gimple_call_num_args (call_stmt) == 2) - { - int val1, val2; - val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1)); - int d2 - = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2); - if (d2 != 2 || val1 != val2) - return NULL; - } + /* ctzll (x) == ctz (x) for unsigned or signed x != 0. */ break; case IFN_FFS: /* ffsll (x) == ffs (x) for unsigned or signed x. */ @@ -2301,6 +2277,21 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, gcc_unreachable (); } + /* For zero the pattern gives the value of the target for VEC_TYPE plus + ADDEND. Punt if the call has another value at zero. */ + if ((ifn == IFN_CLZ || ifn == IFN_CTZ) + && gimple_call_internal_p (call_stmt) + && gimple_call_num_args (call_stmt) == 2) + { + int val; + int d = (ifn == IFN_CLZ + ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val) + : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val)); + if (d != 2 + || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend) + return NULL; + } + /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */ bool supported = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
