On Tue, 13 Oct 2026, [email protected] wrote:
> From: Kyrylo Tkachov <[email protected]>
>
> vect_recog_popcount_clz_ctz_ffs_pattern checks the second argument of a
> two-argument .CLZ or .CTZ against the value of the target. It does
> this only when the precisions of the argument and the result differ.
> For equal precisions it builds the pattern call with the value of the
> target. On AArch64
>
> _5 = .CLZ (_1, 5);
> _9 = (unsigned int) _5;
>
> becomes
>
> vect_patt_18.11_23 = .CLZ (vect__1.10_22, 32);
>
> and zero lanes give 32 instead of 5. The expansion implements any value
> at zero of a scalar call, so such scalar calls are valid.
>
> Do the check after the switch, for all precisions. The calls of the
> example then stay scalar.
>
> Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux.
I think that's OK (correct), but can we do better? We should
be able to, since at least for 'unsupported' we dispatch to
vect_recog_ctz_ffs_pattern which eventually does
if (defined_at_zero
&& (!defined_at_zero_new || val != val_cmp))
{
append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
rhs_oprnd = gimple_call_arg (call_stmt, 0);
rhs_type = TREE_TYPE (rhs_oprnd);
tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
pattern_stmt = gimple_build_assign (cmp, NE_EXPR, rhs_oprnd,
build_zero_cst (rhs_type));
append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
truth_type_for (vec_type), rhs_type);
pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
new_var,
build_int_cst (lhs_type, val));
> gcc/ChangeLog:
>
> * tree-vect-patterns.cc (vect_recog_popcount_clz_ctz_ffs_pattern):
> Check the value at zero of a two-argument .CLZ or .CTZ for all
> precisions.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c: New test.
>
> Signed-off-by: Kyrylo Tkachov <[email protected]>
> ---
> .../vect/bb-slp-clz-ctz-value-at-zero-1.c | 182 ++++++++++++++++++
> gcc/tree-vect-patterns.cc | 43 ++---
> 2 files changed, 199 insertions(+), 26 deletions(-)
> create mode 100644 gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
>
> diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
> b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
> new file mode 100644
> index 00000000000..64404dc01e0
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
> @@ -0,0 +1,182 @@
> +/* Lanes of .CLZ and .CTZ whose value at zero is not the value of the
> + target must keep that value. */
> +/* { dg-require-effective-target int32 } */
> +/* { dg-require-effective-target clz_ifn } */
> +/* { dg-require-effective-target ctz_ifn } */
> +/* { dg-additional-options "-fgimple -fdump-tree-optimized" } */
> +
> +#include "tree-vect.h"
> +
> +unsigned int a[4];
> +int r[4];
> +unsigned int u[4];
> +
> +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
> +clz5 (void)
> +{
> + unsigned int _1;
> + unsigned int _2;
> + unsigned int _3;
> + unsigned int _4;
> + int _5;
> + int _6;
> + int _7;
> + int _8;
> +
> + __BB(2):
> + _1 = a[0];
> + _2 = a[1];
> + _3 = a[2];
> + _4 = a[3];
> + _5 = .CLZ (_1, 5);
> + _6 = .CLZ (_2, 5);
> + _7 = .CLZ (_3, 5);
> + _8 = .CLZ (_4, 5);
> + r[0] = _5;
> + r[1] = _6;
> + r[2] = _7;
> + r[3] = _8;
> + return;
> +}
> +
> +/* The conversion makes the call part of a vectorizer pattern. */
> +
> +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
> +clz5u (void)
> +{
> + unsigned int _1;
> + unsigned int _2;
> + unsigned int _3;
> + unsigned int _4;
> + int _5;
> + int _6;
> + int _7;
> + int _8;
> + unsigned int _9;
> + unsigned int _10;
> + unsigned int _11;
> + unsigned int _12;
> +
> + __BB(2):
> + _1 = a[0];
> + _2 = a[1];
> + _3 = a[2];
> + _4 = a[3];
> + _5 = .CLZ (_1, 5);
> + _6 = .CLZ (_2, 5);
> + _7 = .CLZ (_3, 5);
> + _8 = .CLZ (_4, 5);
> + _9 = (unsigned int) _5;
> + _10 = (unsigned int) _6;
> + _11 = (unsigned int) _7;
> + _12 = (unsigned int) _8;
> + u[0] = _9;
> + u[1] = _10;
> + u[2] = _11;
> + u[3] = _12;
> + return;
> +}
> +
> +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
> +ctz5u (void)
> +{
> + unsigned int _1;
> + unsigned int _2;
> + unsigned int _3;
> + unsigned int _4;
> + int _5;
> + int _6;
> + int _7;
> + int _8;
> + unsigned int _9;
> + unsigned int _10;
> + unsigned int _11;
> + unsigned int _12;
> +
> + __BB(2):
> + _1 = a[0];
> + _2 = a[1];
> + _3 = a[2];
> + _4 = a[3];
> + _5 = .CTZ (_1, 5);
> + _6 = .CTZ (_2, 5);
> + _7 = .CTZ (_3, 5);
> + _8 = .CTZ (_4, 5);
> + _9 = (unsigned int) _5;
> + _10 = (unsigned int) _6;
> + _11 = (unsigned int) _7;
> + _12 = (unsigned int) _8;
> + u[0] = _9;
> + u[1] = _10;
> + u[2] = _11;
> + u[3] = _12;
> + return;
> +}
> +
> +/* 32 is the value of the target on AArch64, so this vectorizes there. */
> +
> +void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
> +clz32u (void)
> +{
> + unsigned int _1;
> + unsigned int _2;
> + unsigned int _3;
> + unsigned int _4;
> + int _5;
> + int _6;
> + int _7;
> + int _8;
> + unsigned int _9;
> + unsigned int _10;
> + unsigned int _11;
> + unsigned int _12;
> +
> + __BB(2):
> + _1 = a[0];
> + _2 = a[1];
> + _3 = a[2];
> + _4 = a[3];
> + _5 = .CLZ (_1, 32);
> + _6 = .CLZ (_2, 32);
> + _7 = .CLZ (_3, 32);
> + _8 = .CLZ (_4, 32);
> + _9 = (unsigned int) _5;
> + _10 = (unsigned int) _6;
> + _11 = (unsigned int) _7;
> + _12 = (unsigned int) _8;
> + u[0] = _9;
> + u[1] = _10;
> + u[2] = _11;
> + u[3] = _12;
> + return;
> +}
> +
> +static void __attribute__((noipa))
> +check (const unsigned int *p, unsigned int e0, unsigned int e1,
> + unsigned int e2, unsigned int e3)
> +{
> + if (p[0] != e0 || p[1] != e1 || p[2] != e2 || p[3] != e3)
> + __builtin_abort ();
> +}
> +
> +int
> +main (void)
> +{
> + check_vect ();
> +
> + /* Write A at run time, so that LTO cannot fold the loads. */
> + a[1] = 1;
> + a[3] = 0x80000000u;
> + clz5 ();
> + check ((const unsigned int *) r, 5, 31, 5, 0);
> + clz5u ();
> + check (u, 5, 31, 5, 0);
> + ctz5u ();
> + check (u, 5, 0, 5, 31);
> + clz32u ();
> + check (u, 32, 31, 32, 0);
> + return 0;
> +}
> +
> +/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 1 "optimized" { target
> aarch64*-*-* } } } */
> +/* { dg-final { scan-tree-dump-not {\.CTZ \(vect_} "optimized" { target
> aarch64*-*-* } } } */
> diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
> index e9d2870feb4..7c435adeb98 100644
> --- a/gcc/tree-vect-patterns.cc
> +++ b/gcc/tree-vect-patterns.cc
> @@ -2260,9 +2260,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> return NULL;
> break;
> case IFN_CLZ:
> - /* clzll (x) == clz (x) + 32 for unsigned x != 0, so ok
> - if it is undefined at zero or if it matches also for the
> - defined value there. */
> + /* clzll (x) == clz (x) + 32 for unsigned x != 0. */
> if (!TYPE_UNSIGNED (unprom_diff.type))
> return NULL;
> if (!type_has_mode_precision_p (lhs_type)
> @@ -2270,31 +2268,9 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> return NULL;
> addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
> - TYPE_PRECISION (lhs_type));
> - if (gimple_call_internal_p (call_stmt)
> - && gimple_call_num_args (call_stmt) == 2)
> - {
> - int val1, val2;
> - val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
> - int d2
> - = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
> - if (d2 != 2 || val1 != val2 + addend)
> - return NULL;
> - }
> break;
> case IFN_CTZ:
> - /* ctzll (x) == ctz (x) for unsigned or signed x != 0, so ok
> - if it is undefined at zero or if it matches also for the
> - defined value there. */
> - if (gimple_call_internal_p (call_stmt)
> - && gimple_call_num_args (call_stmt) == 2)
> - {
> - int val1, val2;
> - val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
> - int d2
> - = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
> - if (d2 != 2 || val1 != val2)
> - return NULL;
> - }
> + /* ctzll (x) == ctz (x) for unsigned or signed x != 0. */
> break;
> case IFN_FFS:
> /* ffsll (x) == ffs (x) for unsigned or signed x. */
> @@ -2303,6 +2279,21 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
> *vinfo,
> gcc_unreachable ();
> }
>
> + /* For zero the pattern gives the value of the target for VEC_TYPE plus
> + ADDEND. Punt if the call has another value at zero. */
> + if ((ifn == IFN_CLZ || ifn == IFN_CTZ)
> + && gimple_call_internal_p (call_stmt)
> + && gimple_call_num_args (call_stmt) == 2)
> + {
> + int val;
> + int d = (ifn == IFN_CLZ
> + ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val)
> + : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val));
> + if (d != 2
> + || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend)
> + return NULL;
> + }
> +
> /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
> bool supported
> = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Stefan Gaiser, Jochen Jaser, Abhinav Puri; (HRB 36809, AG Nuernberg)
