https://gcc.gnu.org/g:17f81875821fff2197a80017aa4b212352d3e9f4
commit r17-6048-g17f81875821fff2197a80017aa4b212352d3e9f4 Author: Kyrylo Tkachov <[email protected]> Date: Wed Nov 4 00:23:54 2026 -0800 match.pd: Look through VIEW_CONVERT_EXPR in the C[LT]Z zero select folds The C[LT]Z zero select folds remove the select in cnd (X != 0, .C?Z ((T) X, C), C) when C is the value of the count at zero. They look through a conversion of X, but not through a VIEW_CONVERT_EXPR. A vector sign change is a VIEW_CONVERT_EXPR, so the folds fail when the vectorizer counts a signed vector as unsigned. The next patch makes it do that. For for (int i = 0; i < 2048; ++i) p[i] = q[i] ? __builtin_ctzll (q[i]) : 64; with long long q, the vectorizer then gives with SVE vect_patt_21 = VIEW_CONVERT_EXPR<vector([2,2]) unsigned long>(vect__4); vect_patt_38 = .CTZ (vect_patt_21, 64); vect_iftmp = VEC_COND_EXPR <mask__22, vect_patt_38, { 64, ... }>; and the select stays in the code as a sel instruction. Add the match int_convert. It matches a conversion between integral types, and a VIEW_CONVERT_EXPR between vector types with the same number of elements that does not change the representation of an element (vector_nop_conversion_p). Thus (T) X is zero where X is zero, and the count gives C there. Use it in the vector forms of the folds. The select then folds into vect_patt_38. Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux. gcc/ChangeLog: * match.pd (int_convert): New match. Use it in the vector forms of the C[LT]Z zero select folds. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sve/clz-ctz-zero-select-1.c: New test. Signed-off-by: Kyrylo Tkachov <[email protected]> Diff: --- gcc/match.pd | 14 ++++++-- .../gcc.target/aarch64/sve/clz-ctz-zero-select-1.c | 40 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/gcc/match.pd b/gcc/match.pd index 82fe9d6824a5..3912f9f7d543 100644 --- a/gcc/match.pd +++ b/gcc/match.pd @@ -163,6 +163,16 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) (view_convert @0) (if (vector_nop_conversion_p (type, TREE_TYPE (@0))))) +/* Match a conversion between integral types, or a view conversion + between vector types that does not change the elements. */ +(match (int_convert @0) + (convert @0) + (if (ANY_INTEGRAL_TYPE_P (type) + && ANY_INTEGRAL_TYPE_P (TREE_TYPE (@0))))) +(match (int_convert @0) + (view_convert @0) + (if (vector_nop_conversion_p (type, TREE_TYPE (@0))))) + /* These are used by gimple_bitwise_inverted_equal_p to simplify detection of BIT_NOT and comparisons. */ (match (bit_not_with_nop @0) @@ -11989,7 +11999,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) (IFN_CLZ @3 @2))))) (for cnd (cond vec_cond) (simplify - (cnd (ne @0 integer_zerop@1) (IFN_CLZ (convert?@3 @0) INTEGER_CST@2) uniform_integer_cst_p@4) + (cnd (ne @0 integer_zerop@1) (IFN_CLZ (int_convert?@3 @0) INTEGER_CST@2) uniform_integer_cst_p@4) (with { int val; internal_fn ifn = IFN_LAST; if (BITINT_TYPE_P (TREE_TYPE (@3))) @@ -12029,7 +12039,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) (IFN_CTZ @3 @2))))) (for cnd (cond vec_cond) (simplify - (cnd (ne @0 integer_zerop@1) (IFN_CTZ (convert?@3 @0) INTEGER_CST@2) uniform_integer_cst_p@4) + (cnd (ne @0 integer_zerop@1) (IFN_CTZ (int_convert?@3 @0) INTEGER_CST@2) uniform_integer_cst_p@4) (with { int val; internal_fn ifn = IFN_LAST; if (BITINT_TYPE_P (TREE_TYPE (@3))) diff --git a/gcc/testsuite/gcc.target/aarch64/sve/clz-ctz-zero-select-1.c b/gcc/testsuite/gcc.target/aarch64/sve/clz-ctz-zero-select-1.c new file mode 100644 index 000000000000..b05433acd8cd --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/clz-ctz-zero-select-1.c @@ -0,0 +1,40 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8.2-a+sve -fdump-tree-optimized" } */ + +/* The vectorizer counts the signed Q[I] as unsigned. The select of + the result for zero must still fold into the count. */ + +void +f1 (long long *p, long long *q) +{ + for (int i = 0; i < 2048; ++i) + p[i] = q[i] ? __builtin_ctzll (q[i]) : 64; +} + +void +f2 (long long *p, long long *q) +{ + for (int i = 0; i < 2048; ++i) + p[i] = q[i] ? __builtin_clzll (q[i]) : 64; +} + +void +f3 (int *p, int *q) +{ + for (int i = 0; i < 2048; ++i) + p[i] = q[i] ? __builtin_ctz (q[i]) : 32; +} + +void +f4 (int *p, int *q) +{ + for (int i = 0; i < 2048; ++i) + p[i] = q[i] ? __builtin_clz (q[i]) : 32; +} + +/* { dg-final { scan-tree-dump-times {\.CTZ \(vect[^,]*, 64\)} 1 "optimized" } } */ +/* { dg-final { scan-tree-dump-times {\.CLZ \(vect[^,]*, 64\)} 1 "optimized" } } */ +/* { dg-final { scan-tree-dump-times {\.CTZ \(vect[^,]*, 32\)} 1 "optimized" } } */ +/* { dg-final { scan-tree-dump-times {\.CLZ \(vect[^,]*, 32\)} 1 "optimized" } } */ +/* { dg-final { scan-tree-dump-not {\.VCOND_MASK} "optimized" } } */ +/* { dg-final { scan-assembler-not {\tsel\t} } } */
