https://gcc.gnu.org/g:3b272406eddd7f82749d67483022c7198cdbfc3e

commit r17-5897-g3b272406eddd7f82749d67483022c7198cdbfc3e
Author: Kyrylo Tkachov <[email protected]>
Date:   Fri Oct 9 08:53:43 2026 -0700

    vect: Check the C[LT]Z value at zero for all precisions
    
    vect_recog_popcount_clz_ctz_ffs_pattern checks the second argument of a
    two-argument .CLZ or .CTZ against the value of the target.  It does
    this only when the precisions of the argument and the result differ.
    For equal precisions it builds the pattern call with the value of the
    target.  On AArch64
    
      _5 = .CLZ (_1, 5);
      _9 = (unsigned int) _5;
    
    becomes
    
      vect_patt_18.11_23 = .CLZ (vect__1.10_22, 32);
    
    and zero lanes give 32 instead of 5.  The expansion implements any value
    at zero of a scalar call, so such scalar calls are valid.
    
    Do the check after the switch, for all precisions.  The calls of the
    example then stay scalar.
    
    Bootstrapped and tested on aarch64-none-linux-gnu and x86_64-linux.
    
    gcc/ChangeLog:
    
            * tree-vect-patterns.cc (vect_recog_popcount_clz_ctz_ffs_pattern):
            Check the value at zero of a two-argument .CLZ or .CTZ for all
            precisions.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c: New test.
    
    Signed-off-by: Kyrylo Tkachov <[email protected]>

Diff:
---
 .../gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c   | 182 +++++++++++++++++++++
 gcc/tree-vect-patterns.cc                          |  43 ++---
 2 files changed, 199 insertions(+), 26 deletions(-)

diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c 
b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
new file mode 100644
index 000000000000..23b5c428bbe2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-clz-ctz-value-at-zero-1.c
@@ -0,0 +1,182 @@
+/* Lanes of .CLZ and .CTZ whose value at zero is not the value of the
+   target must keep that value.  */
+/* { dg-require-effective-target int32 } */
+/* { dg-require-effective-target clzsi } */
+/* { dg-require-effective-target ctzsi } */
+/* { dg-additional-options "-fgimple -fdump-tree-optimized" } */
+
+#include "tree-vect.h"
+
+unsigned int a[4];
+int r[4];
+unsigned int u[4];
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+clz5 (void)
+{
+  unsigned int _1;
+  unsigned int _2;
+  unsigned int _3;
+  unsigned int _4;
+  int _5;
+  int _6;
+  int _7;
+  int _8;
+
+  __BB(2):
+  _1 = a[0];
+  _2 = a[1];
+  _3 = a[2];
+  _4 = a[3];
+  _5 = .CLZ (_1, 5);
+  _6 = .CLZ (_2, 5);
+  _7 = .CLZ (_3, 5);
+  _8 = .CLZ (_4, 5);
+  r[0] = _5;
+  r[1] = _6;
+  r[2] = _7;
+  r[3] = _8;
+  return;
+}
+
+/* The conversion makes the call part of a vectorizer pattern.  */
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+clz5u (void)
+{
+  unsigned int _1;
+  unsigned int _2;
+  unsigned int _3;
+  unsigned int _4;
+  int _5;
+  int _6;
+  int _7;
+  int _8;
+  unsigned int _9;
+  unsigned int _10;
+  unsigned int _11;
+  unsigned int _12;
+
+  __BB(2):
+  _1 = a[0];
+  _2 = a[1];
+  _3 = a[2];
+  _4 = a[3];
+  _5 = .CLZ (_1, 5);
+  _6 = .CLZ (_2, 5);
+  _7 = .CLZ (_3, 5);
+  _8 = .CLZ (_4, 5);
+  _9 = (unsigned int) _5;
+  _10 = (unsigned int) _6;
+  _11 = (unsigned int) _7;
+  _12 = (unsigned int) _8;
+  u[0] = _9;
+  u[1] = _10;
+  u[2] = _11;
+  u[3] = _12;
+  return;
+}
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+ctz5u (void)
+{
+  unsigned int _1;
+  unsigned int _2;
+  unsigned int _3;
+  unsigned int _4;
+  int _5;
+  int _6;
+  int _7;
+  int _8;
+  unsigned int _9;
+  unsigned int _10;
+  unsigned int _11;
+  unsigned int _12;
+
+  __BB(2):
+  _1 = a[0];
+  _2 = a[1];
+  _3 = a[2];
+  _4 = a[3];
+  _5 = .CTZ (_1, 5);
+  _6 = .CTZ (_2, 5);
+  _7 = .CTZ (_3, 5);
+  _8 = .CTZ (_4, 5);
+  _9 = (unsigned int) _5;
+  _10 = (unsigned int) _6;
+  _11 = (unsigned int) _7;
+  _12 = (unsigned int) _8;
+  u[0] = _9;
+  u[1] = _10;
+  u[2] = _11;
+  u[3] = _12;
+  return;
+}
+
+/* 32 is the value of the target on AArch64, so this vectorizes there.  */
+
+void __GIMPLE (ssa,startwith ("fix_loops")) __attribute__((noipa))
+clz32u (void)
+{
+  unsigned int _1;
+  unsigned int _2;
+  unsigned int _3;
+  unsigned int _4;
+  int _5;
+  int _6;
+  int _7;
+  int _8;
+  unsigned int _9;
+  unsigned int _10;
+  unsigned int _11;
+  unsigned int _12;
+
+  __BB(2):
+  _1 = a[0];
+  _2 = a[1];
+  _3 = a[2];
+  _4 = a[3];
+  _5 = .CLZ (_1, 32);
+  _6 = .CLZ (_2, 32);
+  _7 = .CLZ (_3, 32);
+  _8 = .CLZ (_4, 32);
+  _9 = (unsigned int) _5;
+  _10 = (unsigned int) _6;
+  _11 = (unsigned int) _7;
+  _12 = (unsigned int) _8;
+  u[0] = _9;
+  u[1] = _10;
+  u[2] = _11;
+  u[3] = _12;
+  return;
+}
+
+static void __attribute__((noipa))
+check (const unsigned int *p, unsigned int e0, unsigned int e1,
+       unsigned int e2, unsigned int e3)
+{
+  if (p[0] != e0 || p[1] != e1 || p[2] != e2 || p[3] != e3)
+    __builtin_abort ();
+}
+
+int
+main (void)
+{
+  check_vect ();
+
+  /* Write A at run time, so that LTO cannot fold the loads.  */
+  a[1] = 1;
+  a[3] = 0x80000000u;
+  clz5 ();
+  check ((const unsigned int *) r, 5, 31, 5, 0);
+  clz5u ();
+  check (u, 5, 31, 5, 0);
+  ctz5u ();
+  check (u, 5, 0, 5, 31);
+  clz32u ();
+  check (u, 32, 31, 32, 0);
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times {\.CLZ \(vect_} 1 "optimized" { target 
aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-not {\.CTZ \(vect_} "optimized" { target 
aarch64*-*-* } } } */
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index d79f625280a6..c706dafd6111 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -2258,9 +2258,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
          return NULL;
        break;
       case IFN_CLZ:
-       /* clzll (x) == clz (x) + 32 for unsigned x != 0, so ok
-          if it is undefined at zero or if it matches also for the
-          defined value there.  */
+       /* clzll (x) == clz (x) + 32 for unsigned x != 0.  */
        if (!TYPE_UNSIGNED (unprom_diff.type))
          return NULL;
        if (!type_has_mode_precision_p (lhs_type)
@@ -2268,31 +2266,9 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
          return NULL;
        addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
                  - TYPE_PRECISION (lhs_type));
-       if (gimple_call_internal_p (call_stmt)
-           && gimple_call_num_args (call_stmt) == 2)
-         {
-           int val1, val2;
-           val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
-           int d2
-             = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
-           if (d2 != 2 || val1 != val2 + addend)
-             return NULL;
-         }
        break;
       case IFN_CTZ:
-       /* ctzll (x) == ctz (x) for unsigned or signed x != 0, so ok
-          if it is undefined at zero or if it matches also for the
-          defined value there.  */
-       if (gimple_call_internal_p (call_stmt)
-           && gimple_call_num_args (call_stmt) == 2)
-         {
-           int val1, val2;
-           val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
-           int d2
-             = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
-           if (d2 != 2 || val1 != val2)
-             return NULL;
-         }
+       /* ctzll (x) == ctz (x) for unsigned or signed x != 0.  */
        break;
       case IFN_FFS:
        /* ffsll (x) == ffs (x) for unsigned or signed x.  */
@@ -2301,6 +2277,21 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
        gcc_unreachable ();
       }
 
+  /* For zero the pattern gives the value of the target for VEC_TYPE plus
+     ADDEND.  Punt if the call has another value at zero.  */
+  if ((ifn == IFN_CLZ || ifn == IFN_CTZ)
+      && gimple_call_internal_p (call_stmt)
+      && gimple_call_num_args (call_stmt) == 2)
+    {
+      int val;
+      int d = (ifn == IFN_CLZ
+              ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val)
+              : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val));
+      if (d != 2
+         || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend)
+       return NULL;
+    }
+
   /* Do it only if the backend has popcount<vector_mode>2 etc. pattern.  */
   bool supported
     = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);

Reply via email to