https://gcc.gnu.org/g:3fe83b6ed0677ecb87ce0bb465a1877135edb072
commit r17-5942-g3fe83b6ed0677ecb87ce0bb465a1877135edb072 Author: Kyrylo Tkachov <[email protected]> Date: Thu Oct 15 05:30:22 2026 -0700 vect: Select the C[LT]Z value at zero in the bit count pattern vect_recog_popcount_clz_ctz_ffs_pattern punts if a two-argument .CLZ or .CTZ has a value at zero that is not the value of the target for the vector mode, plus the addend of a widened CLZ. vect_recog_ctz_ffs_pattern already handles such a difference with a select. Do the same here: count with the value of the target, then select the value at zero of the call for zero lanes. If the count needs vect_recog_ctz_ffs_pattern, give it the value at zero of the call, so that it adds the select. On AArch64 _5 = .CTZ (_1, 5); _9 = (unsigned int) _5; now vectorizes to vect_patt_18.27_32 = .CTZ (vect__1.25_30, 32); mask_patt_19.26_31 = vect__1.25_30 != { 0, 0, 0, 0 }; vect_patt_20.28_33 = .VCOND_MASK (mask_patt_19.26_31, vect_patt_18.27_32, { 5, 5, 5, 5 }); Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux. gcc/ChangeLog: * tree-vect-patterns.cc (vect_recog_popcount_clz_ctz_ffs_pattern): Select the value at zero of a two-argument .CLZ or .CTZ if it is not the value of the target for the vector mode. gcc/testsuite/ChangeLog: * gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c: Expect vector counts with a select on aarch64. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- .../gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c | 8 ++++--- gcc/tree-vect-patterns.cc | 26 ++++++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c index 23b5c428bbe2..94e63a4a9c45 100644 --- a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c @@ -39,7 +39,8 @@ clz5 (void) return; } -/* The conversion makes the call part of a vectorizer pattern. */ +/* The conversion makes the call part of a vectorizer pattern, which + selects 5 for zero lanes. */ void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa)) clz5u (void) @@ -178,5 +179,6 @@ main (void) return 0; } -/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 1 "optimized" { target aarch64*-*-* } } } */ -/* { dg-final { scan-tree-dump-not {\.CTZ \(vect_} "optimized" { target aarch64*-*-* } } } */ +/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 2 "optimized" { target aarch64*-*-* } } } */ +/* { dg-final { scan-tree-dump-times {\.CTZ \(vect_} 1 "optimized" { target aarch64*-*-* } } } */ +/* { dg-final { scan-tree-dump-times {\.VCOND_MASK \(} 2 "optimized" { target aarch64*-*-* } } } */ diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc index c706dafd6111..a69b998e6448 100644 --- a/gcc/tree-vect-patterns.cc +++ b/gcc/tree-vect-patterns.cc @@ -2278,7 +2278,8 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, } /* For zero the pattern gives the value of the target for VEC_TYPE plus - ADDEND. Punt if the call has another value at zero. */ + ADDEND. If the call has another value at zero, select it. */ + tree zero_val = NULL_TREE; if ((ifn == IFN_CLZ || ifn == IFN_CTZ) && gimple_call_internal_p (call_stmt) && gimple_call_num_args (call_stmt) == 2) @@ -2289,7 +2290,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val)); if (d != 2 || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend) - return NULL; + zero_val = gimple_call_arg (call_stmt, 1); } /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */ @@ -2352,6 +2353,12 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val) == 2) arg2 = build_int_cst (integer_type_node, val); + /* vect_recog_ctz_ffs_pattern selects the value at zero itself. */ + if (zero_val && !supported) + { + arg2 = zero_val; + zero_val = NULL_TREE; + } if (arg2) pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2); else @@ -2391,6 +2398,21 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, else if (cast_stmt) append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type); + if (zero_val) + { + append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type); + tree op = unprom_diff.op; + tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL); + gimple *cmp_stmt = gimple_build_assign (cmp, NE_EXPR, op, + build_zero_cst (TREE_TYPE (op))); + append_pattern_def_seq (vinfo, stmt_vinfo, cmp_stmt, + truth_type_for (vec_type), TREE_TYPE (op)); + tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL); + pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp, + gimple_get_lhs (pattern_stmt), + fold_convert (lhs_type, zero_val)); + } + if (cast_stmt) { append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
