https://gcc.gnu.org/g:3fe83b6ed0677ecb87ce0bb465a1877135edb072

commit r17-5942-g3fe83b6ed0677ecb87ce0bb465a1877135edb072
Author: Kyrylo Tkachov <[email protected]>
Date:   Thu Oct 15 05:30:22 2026 -0700

    vect: Select the C[LT]Z value at zero in the bit count pattern
    
    vect_recog_popcount_clz_ctz_ffs_pattern punts if a two-argument .CLZ or
    .CTZ has a value at zero that is not the value of the target for the
    vector mode, plus the addend of a widened CLZ.
    vect_recog_ctz_ffs_pattern already handles such a difference with a
    select.  Do the same here: count with the value of the target, then
    select the value at zero of the call for zero lanes.  If the count
    needs vect_recog_ctz_ffs_pattern, give it the value at zero of the
    call, so that it adds the select.  On AArch64
    
      _5 = .CTZ (_1, 5);
      _9 = (unsigned int) _5;
    
    now vectorizes to
    
      vect_patt_18.27_32 = .CTZ (vect__1.25_30, 32);
      mask_patt_19.26_31 = vect__1.25_30 != { 0, 0, 0, 0 };
      vect_patt_20.28_33 = .VCOND_MASK (mask_patt_19.26_31,
                                        vect_patt_18.27_32, { 5, 5, 5, 5 });
    
    Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux.
    
    gcc/ChangeLog:
    
            * tree-vect-patterns.cc (vect_recog_popcount_clz_ctz_ffs_pattern):
            Select the value at zero of a two-argument .CLZ or .CTZ if it is not
            the value of the target for the vector mode.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c: Expect vector counts
            with a select on aarch64.
    
    Signed-off-by: Kyrylo Tkachov <[email protected]>

Diff:
---
 .../gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c   |  8 ++++---
 gcc/tree-vect-patterns.cc                          | 26 ++++++++++++++++++++--
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c 
b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
index 23b5c428bbe2..94e63a4a9c45 100644
--- a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
@@ -39,7 +39,8 @@ clz5 (void)
   return;
 }
 
-/* The conversion makes the call part of a vectorizer pattern.  */
+/* The conversion makes the call part of a vectorizer pattern, which
+   selects 5 for zero lanes.  */
 
 void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
 clz5u (void)
@@ -178,5 +179,6 @@ main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 1 "optimized" { target 
aarch64*-*-* } } } */
-/* { dg-final { scan-tree-dump-not {\.CTZ \(vect_} "optimized" { target 
aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 2 "optimized" { target 
aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-times {\.CTZ \(vect_} 1 "optimized" { target 
aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-times {\.VCOND_MASK \(} 2 "optimized" { target 
aarch64*-*-* } } } */
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index c706dafd6111..a69b998e6448 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -2278,7 +2278,8 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
       }
 
   /* For zero the pattern gives the value of the target for VEC_TYPE plus
-     ADDEND.  Punt if the call has another value at zero.  */
+     ADDEND.  If the call has another value at zero, select it.  */
+  tree zero_val = NULL_TREE;
   if ((ifn == IFN_CLZ || ifn == IFN_CTZ)
       && gimple_call_internal_p (call_stmt)
       && gimple_call_num_args (call_stmt) == 2)
@@ -2289,7 +2290,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
               : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val));
       if (d != 2
          || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend)
-       return NULL;
+       zero_val = gimple_call_arg (call_stmt, 1);
     }
 
   /* Do it only if the backend has popcount<vector_mode>2 etc. pattern.  */
@@ -2352,6 +2353,12 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
           && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
                                         val) == 2)
     arg2 = build_int_cst (integer_type_node, val);
+  /* vect_recog_ctz_ffs_pattern selects the value at zero itself.  */
+  if (zero_val && !supported)
+    {
+      arg2 = zero_val;
+      zero_val = NULL_TREE;
+    }
   if (arg2)
     pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
   else
@@ -2391,6 +2398,21 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
   else if (cast_stmt)
     append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type);
 
+  if (zero_val)
+    {
+      append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
+      tree op = unprom_diff.op;
+      tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
+      gimple *cmp_stmt = gimple_build_assign (cmp, NE_EXPR, op,
+                                             build_zero_cst (TREE_TYPE (op)));
+      append_pattern_def_seq (vinfo, stmt_vinfo, cmp_stmt,
+                             truth_type_for (vec_type), TREE_TYPE (op));
+      tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
+      pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
+                                 gimple_get_lhs (pattern_stmt),
+                                 fold_convert (lhs_type, zero_val));
+    }
+
   if (cast_stmt)
     {
       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);

Reply via email to