https://gcc.gnu.org/g:f088a08addbb4d50ca88d64a7fd94234054f2f47

commit r17-5840-gf088a08addbb4d50ca88d64a7fd94234054f2f47
Author: Igor Shevlyakov <[email protected]>
Date:   Sat Oct 24 13:04:25 2026 -0700

    match.pd: Fold same-width signed vector averages
    
    (x + y) >> 1 is IFN_AVG_FLOOR (x, y) and ((x + y) + 1) >> 1 is
    IFN_AVG_CEIL (x, y) whenever the sum does not wrap.  The existing
    IFN_AVG_FLOOR fold for vectors proves that from leading zero bits of x
    and y.  For a signed type whose overflow is undefined the sum cannot
    wrap anyway, so code such as the column pass of a 4x4 DCT butterfly,
    
      for (int i = 0; i < n; i++)
        r[i] = (a[i] + b[i] + c[i] + d[i] + 1) >> 1;
    
    is vectorized as two adds and an arithmetic shift per vector where one
    add and a rounding halving add (srhadd on AArch64) would do.
    vect_recog_average_pattern only handles the narrowing form, where the
    sum is computed in a wider type than x and y.
    
    Fold (x + y) >> 1 without looking at the bits of x and y when overflow
    is undefined, and add the same fold of ((x + y) + 1) >> 1 to
    IFN_AVG_CEIL, with the same conditions.
    
    Assisted-by: Claude Opus 5.5
    
    gcc/ChangeLog:
    
            * match.pd ((x + y) >> 1 -> IFN_AVG_FLOOR (x, y)): Also fold
            when overflow is undefined.
            (((x + y) + 1) >> 1 -> IFN_AVG_CEIL (x, y)): New fold.
    
    gcc/testsuite/ChangeLog:
    
            * gcc.dg/vect/vect-avg-17.c: New test.
            * gcc.target/aarch64/vect_hadd_same_width_1.c: New test.
            * gcc.target/aarch64/vect_hadd_same_width_2.c: New test.
    
    Signed-off-by: Igor Shevlyakov <[email protected]>

Diff:
---
 gcc/match.pd                                       | 33 +++++---
 gcc/testsuite/gcc.dg/vect/vect-avg-17.c            | 87 ++++++++++++++++++++++
 .../gcc.target/aarch64/vect_hadd_same_width_1.c    | 47 ++++++++++++
 .../gcc.target/aarch64/vect_hadd_same_width_2.c    |  7 ++
 4 files changed, 162 insertions(+), 12 deletions(-)

diff --git a/gcc/match.pd b/gcc/match.pd
index f25952c6fd88..70f956a692f1 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -2643,23 +2643,32 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
   (avg @0 @0)
   @0))
 
- /* Fold ((x + y) >> 1 into IFN_AVG_FLOOR (x, y) if x and y are vectors in
-    which x + y cannot wrap: each element is known to have at least one
-    leading zero bit, or two if the type is signed and its overflow is
-    defined, since the sum of two nonnegative values can then wrap to a
-    negative one.  */
+ /* Fold (x + y) >> 1 into IFN_AVG_FLOOR (x, y) and ((x + y) + 1) >> 1
+    into IFN_AVG_CEIL (x, y) if x and y are vectors in which the sum
+    cannot wrap: overflow is undefined, or each element of x and y is
+    known to have at least one leading zero bit, or two if the type is
+    signed, since the sum of two nonnegative values can then wrap to a
+    negative one.  The IFN_AVG_CEIL fold comes first because
+    ((x + y) + 1) >> 1 is also (x' + 1) >> 1.  */
+(simplify
+ (rshift (plus:s (plus:cs @0 @1) integer_onep) integer_onep)
+ (if (VECTOR_TYPE_P (type)
+      && direct_internal_fn_supported_p (IFN_AVG_CEIL, type,
+                                        OPTIMIZE_FOR_BOTH))
+  (with { int min_clz = TYPE_UNSIGNED (type) ? 1 : 2; }
+   (if (TYPE_OVERFLOW_UNDEFINED (type)
+       || (wi::clz (get_nonzero_bits (@0)) >= min_clz
+           && wi::clz (get_nonzero_bits (@1)) >= min_clz))
+    (IFN_AVG_CEIL @0 @1)))))
 (simplify
  (rshift (plus:cs @0 @1) integer_onep)
  (if (VECTOR_TYPE_P (type)
       && direct_internal_fn_supported_p (IFN_AVG_FLOOR, type,
                                         OPTIMIZE_FOR_BOTH))
-  (with
-   {
-     int min_clz = (TYPE_UNSIGNED (type) || TYPE_OVERFLOW_UNDEFINED (type)
-                   ? 1 : 2);
-   }
-   (if (wi::clz (get_nonzero_bits (@0)) >= min_clz
-       && wi::clz (get_nonzero_bits (@1)) >= min_clz)
+  (with { int min_clz = TYPE_UNSIGNED (type) ? 1 : 2; }
+   (if (TYPE_OVERFLOW_UNDEFINED (type)
+       || (wi::clz (get_nonzero_bits (@0)) >= min_clz
+           && wi::clz (get_nonzero_bits (@1)) >= min_clz))
     (IFN_AVG_FLOOR @0 @1)))))
 #endif
 
diff --git a/gcc/testsuite/gcc.dg/vect/vect-avg-17.c 
b/gcc/testsuite/gcc.dg/vect/vect-avg-17.c
new file mode 100644
index 000000000000..3fab78ea1000
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-avg-17.c
@@ -0,0 +1,87 @@
+/* { dg-additional-options "-O3 -Wno-psabi" } */
+/* { dg-require-effective-target vect_int } */
+
+#include "tree-vect.h"
+
+#define N 128
+
+int a[N], b[N], c[N], d[N], r[N];
+
+void __attribute__ ((noipa))
+floor2 (void)
+{
+  for (int i = 0; i < N; ++i)
+    r[i] = (a[i] + b[i]) >> 1;
+}
+
+void __attribute__ ((noipa))
+ceil2 (void)
+{
+  for (int i = 0; i < N; ++i)
+    r[i] = (a[i] + b[i] + 1) >> 1;
+}
+
+void __attribute__ ((noipa))
+ceil4 (void)
+{
+  for (int i = 0; i < N; ++i)
+    r[i] = (a[i] + b[i] + c[i] + d[i] + 1) >> 1;
+}
+
+typedef short v8hi __attribute__ ((vector_size (16)));
+
+v8hi __attribute__ ((noipa))
+floor_v8hi (v8hi x, v8hi y)
+{
+  return (x + y) >> 1;
+}
+
+v8hi __attribute__ ((noipa))
+ceil_v8hi (v8hi x, v8hi y)
+{
+  return (x + y + 1) >> 1;
+}
+
+int
+main (void)
+{
+  check_vect ();
+
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    {
+      a[i] = (i * 37 % 101 - 50) * 1000003;
+      b[i] = (i * 53 % 97 - 48) * 999983 + (i & 1);
+      c[i] = -(i * 29 % 89) * 1000033 - 1;
+      d[i] = (i * 17 % 83 - 41) * 999979;
+    }
+
+  floor2 ();
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    if (r[i] != (a[i] + b[i]) >> 1)
+      __builtin_abort ();
+
+  ceil2 ();
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    if (r[i] != (a[i] + b[i] + 1) >> 1)
+      __builtin_abort ();
+
+  ceil4 ();
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    if (r[i] != (a[i] + b[i] + c[i] + d[i] + 1) >> 1)
+      __builtin_abort ();
+
+  v8hi x = { -32767, -3, -3, 5, 32766, 0, -1, 7 };
+  v8hi y = { -1, -2, 4, 6, -32767, -1, -1, -8 };
+  v8hi f = floor_v8hi (x, y);
+  v8hi g = ceil_v8hi (x, y);
+#pragma GCC novector
+  for (int i = 0; i < 8; ++i)
+    if (f[i] != (x[i] + y[i]) >> 1 || g[i] != (x[i] + y[i] + 1) >> 1)
+      __builtin_abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_1.c 
b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_1.c
new file mode 100644
index 000000000000..b6ca8d4309cf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_1.c
@@ -0,0 +1,47 @@
+/* Same-width averages of signed types whose overflow is undefined.  */
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
+
+typedef short v8hi __attribute__ ((vector_size (16)));
+typedef signed char v16qi __attribute__ ((vector_size (16)));
+typedef unsigned short v8hu __attribute__ ((vector_size (16)));
+
+void
+ceil4 (int *restrict r, int *restrict a, int *restrict b, int *restrict c,
+       int *restrict d, int n)
+{
+  for (int i = 0; i < n; i++)
+    r[i] = (a[i] + b[i] + c[i] + d[i] + 1) >> 1;
+}
+
+void
+floor2 (int *restrict r, int *restrict a, int *restrict b, int n)
+{
+  for (int i = 0; i < n; i++)
+    r[i] = (a[i] + b[i]) >> 1;
+}
+
+v8hi ceil_v8hi (v8hi a, v8hi b) { return (a + b + 1) >> 1; }
+v8hi floor_v8hi (v8hi a, v8hi b) { return (a + b) >> 1; }
+v16qi ceil_v16qi (v16qi a, v16qi b) { return (a + b + 1) >> 1; }
+v16qi floor_v16qi (v16qi a, v16qi b) { return (a + b) >> 1; }
+
+/* Unsigned sums can wrap.  */
+
+void
+ceil_unsigned (unsigned *restrict r, unsigned *restrict a,
+              unsigned *restrict b, int n)
+{
+  for (int i = 0; i < n; i++)
+    r[i] = (a[i] + b[i] + 1) >> 1;
+}
+
+v8hu floor_v8hu (v8hu a, v8hu b) { return (a + b) >> 1; }
+
+/* { dg-final { scan-assembler {\tsrhadd\tv[0-9]+\.4s,} } } */
+/* { dg-final { scan-assembler {\tshadd\tv[0-9]+\.4s,} } } */
+/* { dg-final { scan-assembler-times {\tsrhadd\tv[0-9]+\.8h,} 1 } } */
+/* { dg-final { scan-assembler-times {\tshadd\tv[0-9]+\.8h,} 1 } } */
+/* { dg-final { scan-assembler-times {\tsrhadd\tv[0-9]+\.16b,} 1 } } */
+/* { dg-final { scan-assembler-times {\tshadd\tv[0-9]+\.16b,} 1 } } */
+/* { dg-final { scan-assembler-not {\tu(r)?hadd\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_2.c 
b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_2.c
new file mode 100644
index 000000000000..441ffdfb8590
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_2.c
@@ -0,0 +1,7 @@
+/* With -fwrapv the sums can wrap, so none of these are averages.  */
+/* { dg-do compile } */
+/* { dg-options "-O3 -fwrapv" } */
+
+#include "vect_hadd_same_width_1.c"
+
+/* { dg-final { scan-assembler-not {\t[su](r)?hadd\t} } } */

Reply via email to