https://gcc.gnu.org/g:e591878900c4142829dbe6701e2aa67da7d96ab2
commit r17-6049-ge591878900c4142829dbe6701e2aa67da7d96ab2 Author: Kyrylo Tkachov <[email protected]> Date: Tue Nov 3 06:26:23 2026 -0800 vect: Pass unsigned arguments to .CLZ, .CTZ and .POPCOUNT .CLZ, .CTZ and .POPCOUNT are defined only for unsigned arguments, but two vectorizer patterns can give them a signed argument. vect_recog_ctz_ffs_pattern implements .CTZ and .FFS with .CTZ, .CLZ or .POPCOUNT. It converts a signed argument to unsigned only for the forms with (X - 1) & ~X. For void f (int *__restrict a, int *__restrict b, int n) { for (int i = 0; i < n; i++) a[i] = __builtin_ffs (b[i]); } on aarch64 the pattern statements are patt_23 = .CTZ (_4, 32); patt_22 = patt_23 + 1; with int _4. vect_recog_popcount_clz_ctz_ffs_pattern looks through conversions of the argument. For __builtin_clzll (q[i]) with long long q[i] it looks through the conversion to unsigned and counts the signed value (gcc.dg/vect/pr109011-1.c with -mavx512cd). Both patterns now convert a signed argument to unsigned. vect_recog_ctz_ffs_pattern does this before all forms, and its checks for reductions move before the conversion, so the pattern does not fail after it adds statements. The example above now gives patt_23 = (unsigned int) _4; patt_22 = .CTZ (patt_23, 32); patt_21 = patt_22 + 1; The conversions do not change the bits. With patch 1/5 the generated code does not change. Patch 5/5 makes the GIMPLE verifier reject signed .CLZ and .CTZ calls. Then gcc.dg/vect/pr109011-*.c and gcc.target/i386/pr109011-*.c test this patch. Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux. gcc/ChangeLog: * tree-vect-patterns.cc (vect_recog_ctz_ffs_pattern): Convert a signed argument to unsigned for all forms. Check for reductions before the conversion. (vect_recog_popcount_clz_ctz_ffs_pattern): Convert a signed argument of .CLZ, .CTZ and .POPCOUNT to unsigned. Signed-off-by: Kyrylo Tkachov <[email protected]> Reviewed-by: Richard Biener <[email protected]> Diff: --- gcc/tree-vect-patterns.cc | 69 +++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc index a69b998e6448..4a458d833cb5 100644 --- a/gcc/tree-vect-patterns.cc +++ b/gcc/tree-vect-patterns.cc @@ -1963,8 +1963,27 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo, if (ifnnew == IFN_LAST) return NULL; + /* Only .FFS (X) = .CTZ (X) + 1 can be part of a reduction, and only + if the result for zero needs no select. */ + if (vect_is_reduction (stmt_vinfo) + && (ifnnew != IFN_CTZ + || (defined_at_zero + && (!defined_at_zero_new || val != val_new + 1)))) + return NULL; + vect_pattern_detected ("vec_recog_ctz_ffs_pattern", call_stmt); + /* .CLZ, .CTZ and .POPCOUNT take an unsigned argument. */ + if (!TYPE_UNSIGNED (rhs_type)) + { + rhs_type = unsigned_type_for (rhs_type); + vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type); + new_var = vect_recog_temp_ssa_var (rhs_type, NULL); + pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd); + append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type); + rhs_oprnd = new_var; + } + val_cmp = val_new; if ((ifnnew == IFN_CLZ && defined_at_zero @@ -1973,26 +1992,12 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo, && val_new == prec) || (ifnnew == IFN_POPCOUNT && ifn == IFN_CTZ)) { - if (vect_is_reduction (stmt_vinfo)) - return NULL; - /* .CTZ (X) = PREC - .CLZ ((X - 1) & ~X) .CTZ (X) = .POPCOUNT ((X - 1) & ~X). */ if (ifnnew == IFN_CLZ) sub = prec; val_cmp = prec; - if (!TYPE_UNSIGNED (rhs_type)) - { - rhs_type = unsigned_type_for (rhs_type); - vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type); - new_var = vect_recog_temp_ssa_var (rhs_type, NULL); - pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd); - append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, - vec_rhs_type); - rhs_oprnd = new_var; - } - tree m1 = vect_recog_temp_ssa_var (rhs_type, NULL); pattern_stmt = gimple_build_assign (m1, PLUS_EXPR, rhs_oprnd, build_int_cst (rhs_type, -1)); @@ -2014,9 +2019,6 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo, } else if (ifnnew == IFN_CLZ) { - if (vect_is_reduction (stmt_vinfo)) - return NULL; - /* .CTZ (X) = (PREC - 1) - .CLZ (X & -X) .FFS (X) = PREC - .CLZ (X & -X). */ sub = prec - (ifn == IFN_CTZ); @@ -2036,9 +2038,6 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo, } else if (ifnnew == IFN_POPCOUNT) { - if (vect_is_reduction (stmt_vinfo)) - return NULL; - /* .CTZ (X) = PREC - .POPCOUNT (X | -X) .FFS (X) = (PREC + 1) - .POPCOUNT (X | -X). */ sub = prec + (ifn == IFN_FFS); @@ -2061,11 +2060,6 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo, /* .FFS (X) = .CTZ (X) + 1. */ add = 1; val_cmp++; - - if (vect_is_reduction (stmt_vinfo) - && defined_at_zero - && (!defined_at_zero_new || val != val_cmp)) - return NULL; } /* Create B = .IFNNEW (A). */ @@ -2331,12 +2325,22 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, call_stmt); tree orig_lhs_type = lhs_type; - gimple *cast_stmt = NULL; if (!type_has_mode_precision_p (lhs_type)) + lhs_type = TREE_TYPE (vec_type); + tree arg_type = (lhs_type != orig_lhs_type + ? lhs_type : TREE_TYPE (unprom_diff.op)); + /* .CLZ, .CTZ and .POPCOUNT take an unsigned argument. */ + if (ifn != IFN_FFS && !TYPE_UNSIGNED (arg_type)) + arg_type = unsigned_type_for (arg_type); + gimple *cast_stmt = NULL; + tree arg_vectype = vec_type; + if (arg_type != TREE_TYPE (unprom_diff.op)) { - lhs_type = TREE_TYPE (vec_type); + arg_vectype = get_vectype_for_scalar_type (vinfo, arg_type); + if (!arg_vectype) + return NULL; cast_stmt - = gimple_build_assign (vect_recog_temp_ssa_var (lhs_type, NULL), + = gimple_build_assign (vect_recog_temp_ssa_var (arg_type, NULL), NOP_EXPR, unprom_diff.op); unprom_diff.op = gimple_assign_lhs (cast_stmt); } @@ -2371,6 +2375,9 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, dump_printf_loc (MSG_NOTE, vect_location, "created pattern stmt: %G", pattern_stmt); + if (cast_stmt && supported) + append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, arg_vectype); + if (addend) { gcc_assert (supported); @@ -2388,15 +2395,13 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, if (pattern_stmt == NULL) return NULL; if (cast_stmt) - append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type); + append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, arg_vectype); if (gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (new_stmt_info)) { gimple_seq *pseq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo); gimple_seq_add_seq_without_update (pseq, seq); } } - else if (cast_stmt) - append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type); if (zero_val) { @@ -2413,7 +2418,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo, fold_convert (lhs_type, zero_val)); } - if (cast_stmt) + if (lhs_type != orig_lhs_type) { append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type); tree ret_var = vect_recog_temp_ssa_var (orig_lhs_type, NULL);
