https://gcc.gnu.org/g:f088a08addbb4d50ca88d64a7fd94234054f2f47
commit r17-5840-gf088a08addbb4d50ca88d64a7fd94234054f2f47 Author: Igor Shevlyakov <[email protected]> Date: Sat Oct 24 13:04:25 2026 -0700 match.pd: Fold same-width signed vector averages (x + y) >> 1 is IFN_AVG_FLOOR (x, y) and ((x + y) + 1) >> 1 is IFN_AVG_CEIL (x, y) whenever the sum does not wrap. The existing IFN_AVG_FLOOR fold for vectors proves that from leading zero bits of x and y. For a signed type whose overflow is undefined the sum cannot wrap anyway, so code such as the column pass of a 4x4 DCT butterfly, for (int i = 0; i < n; i++) r[i] = (a[i] + b[i] + c[i] + d[i] + 1) >> 1; is vectorized as two adds and an arithmetic shift per vector where one add and a rounding halving add (srhadd on AArch64) would do. vect_recog_average_pattern only handles the narrowing form, where the sum is computed in a wider type than x and y. Fold (x + y) >> 1 without looking at the bits of x and y when overflow is undefined, and add the same fold of ((x + y) + 1) >> 1 to IFN_AVG_CEIL, with the same conditions. Assisted-by: Claude Opus 5.5 gcc/ChangeLog: * match.pd ((x + y) >> 1 -> IFN_AVG_FLOOR (x, y)): Also fold when overflow is undefined. (((x + y) + 1) >> 1 -> IFN_AVG_CEIL (x, y)): New fold. gcc/testsuite/ChangeLog: * gcc.dg/vect/vect-avg-17.c: New test. * gcc.target/aarch64/vect_hadd_same_width_1.c: New test. * gcc.target/aarch64/vect_hadd_same_width_2.c: New test. Signed-off-by: Igor Shevlyakov <[email protected]> Diff: --- gcc/match.pd | 33 +++++--- gcc/testsuite/gcc.dg/vect/vect-avg-17.c | 87 ++++++++++++++++++++++ .../gcc.target/aarch64/vect_hadd_same_width_1.c | 47 ++++++++++++ .../gcc.target/aarch64/vect_hadd_same_width_2.c | 7 ++ 4 files changed, 162 insertions(+), 12 deletions(-) diff --git a/gcc/match.pd b/gcc/match.pd index f25952c6fd88..70f956a692f1 100644 --- a/gcc/match.pd +++ b/gcc/match.pd @@ -2643,23 +2643,32 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) (avg @0 @0) @0)) - /* Fold ((x + y) >> 1 into IFN_AVG_FLOOR (x, y) if x and y are vectors in - which x + y cannot wrap: each element is known to have at least one - leading zero bit, or two if the type is signed and its overflow is - defined, since the sum of two nonnegative values can then wrap to a - negative one. */ + /* Fold (x + y) >> 1 into IFN_AVG_FLOOR (x, y) and ((x + y) + 1) >> 1 + into IFN_AVG_CEIL (x, y) if x and y are vectors in which the sum + cannot wrap: overflow is undefined, or each element of x and y is + known to have at least one leading zero bit, or two if the type is + signed, since the sum of two nonnegative values can then wrap to a + negative one. The IFN_AVG_CEIL fold comes first because + ((x + y) + 1) >> 1 is also (x' + 1) >> 1. */ +(simplify + (rshift (plus:s (plus:cs @0 @1) integer_onep) integer_onep) + (if (VECTOR_TYPE_P (type) + && direct_internal_fn_supported_p (IFN_AVG_CEIL, type, + OPTIMIZE_FOR_BOTH)) + (with { int min_clz = TYPE_UNSIGNED (type) ? 1 : 2; } + (if (TYPE_OVERFLOW_UNDEFINED (type) + || (wi::clz (get_nonzero_bits (@0)) >= min_clz + && wi::clz (get_nonzero_bits (@1)) >= min_clz)) + (IFN_AVG_CEIL @0 @1))))) (simplify (rshift (plus:cs @0 @1) integer_onep) (if (VECTOR_TYPE_P (type) && direct_internal_fn_supported_p (IFN_AVG_FLOOR, type, OPTIMIZE_FOR_BOTH)) - (with - { - int min_clz = (TYPE_UNSIGNED (type) || TYPE_OVERFLOW_UNDEFINED (type) - ? 1 : 2); - } - (if (wi::clz (get_nonzero_bits (@0)) >= min_clz - && wi::clz (get_nonzero_bits (@1)) >= min_clz) + (with { int min_clz = TYPE_UNSIGNED (type) ? 1 : 2; } + (if (TYPE_OVERFLOW_UNDEFINED (type) + || (wi::clz (get_nonzero_bits (@0)) >= min_clz + && wi::clz (get_nonzero_bits (@1)) >= min_clz)) (IFN_AVG_FLOOR @0 @1))))) #endif diff --git a/gcc/testsuite/gcc.dg/vect/vect-avg-17.c b/gcc/testsuite/gcc.dg/vect/vect-avg-17.c new file mode 100644 index 000000000000..3fab78ea1000 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-avg-17.c @@ -0,0 +1,87 @@ +/* { dg-additional-options "-O3 -Wno-psabi" } */ +/* { dg-require-effective-target vect_int } */ + +#include "tree-vect.h" + +#define N 128 + +int a[N], b[N], c[N], d[N], r[N]; + +void __attribute__ ((noipa)) +floor2 (void) +{ + for (int i = 0; i < N; ++i) + r[i] = (a[i] + b[i]) >> 1; +} + +void __attribute__ ((noipa)) +ceil2 (void) +{ + for (int i = 0; i < N; ++i) + r[i] = (a[i] + b[i] + 1) >> 1; +} + +void __attribute__ ((noipa)) +ceil4 (void) +{ + for (int i = 0; i < N; ++i) + r[i] = (a[i] + b[i] + c[i] + d[i] + 1) >> 1; +} + +typedef short v8hi __attribute__ ((vector_size (16))); + +v8hi __attribute__ ((noipa)) +floor_v8hi (v8hi x, v8hi y) +{ + return (x + y) >> 1; +} + +v8hi __attribute__ ((noipa)) +ceil_v8hi (v8hi x, v8hi y) +{ + return (x + y + 1) >> 1; +} + +int +main (void) +{ + check_vect (); + +#pragma GCC novector + for (int i = 0; i < N; ++i) + { + a[i] = (i * 37 % 101 - 50) * 1000003; + b[i] = (i * 53 % 97 - 48) * 999983 + (i & 1); + c[i] = -(i * 29 % 89) * 1000033 - 1; + d[i] = (i * 17 % 83 - 41) * 999979; + } + + floor2 (); +#pragma GCC novector + for (int i = 0; i < N; ++i) + if (r[i] != (a[i] + b[i]) >> 1) + __builtin_abort (); + + ceil2 (); +#pragma GCC novector + for (int i = 0; i < N; ++i) + if (r[i] != (a[i] + b[i] + 1) >> 1) + __builtin_abort (); + + ceil4 (); +#pragma GCC novector + for (int i = 0; i < N; ++i) + if (r[i] != (a[i] + b[i] + c[i] + d[i] + 1) >> 1) + __builtin_abort (); + + v8hi x = { -32767, -3, -3, 5, 32766, 0, -1, 7 }; + v8hi y = { -1, -2, 4, 6, -32767, -1, -1, -8 }; + v8hi f = floor_v8hi (x, y); + v8hi g = ceil_v8hi (x, y); +#pragma GCC novector + for (int i = 0; i < 8; ++i) + if (f[i] != (x[i] + y[i]) >> 1 || g[i] != (x[i] + y[i] + 1) >> 1) + __builtin_abort (); + + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_1.c b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_1.c new file mode 100644 index 000000000000..b6ca8d4309cf --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_1.c @@ -0,0 +1,47 @@ +/* Same-width averages of signed types whose overflow is undefined. */ +/* { dg-do compile } */ +/* { dg-options "-O3" } */ + +typedef short v8hi __attribute__ ((vector_size (16))); +typedef signed char v16qi __attribute__ ((vector_size (16))); +typedef unsigned short v8hu __attribute__ ((vector_size (16))); + +void +ceil4 (int *restrict r, int *restrict a, int *restrict b, int *restrict c, + int *restrict d, int n) +{ + for (int i = 0; i < n; i++) + r[i] = (a[i] + b[i] + c[i] + d[i] + 1) >> 1; +} + +void +floor2 (int *restrict r, int *restrict a, int *restrict b, int n) +{ + for (int i = 0; i < n; i++) + r[i] = (a[i] + b[i]) >> 1; +} + +v8hi ceil_v8hi (v8hi a, v8hi b) { return (a + b + 1) >> 1; } +v8hi floor_v8hi (v8hi a, v8hi b) { return (a + b) >> 1; } +v16qi ceil_v16qi (v16qi a, v16qi b) { return (a + b + 1) >> 1; } +v16qi floor_v16qi (v16qi a, v16qi b) { return (a + b) >> 1; } + +/* Unsigned sums can wrap. */ + +void +ceil_unsigned (unsigned *restrict r, unsigned *restrict a, + unsigned *restrict b, int n) +{ + for (int i = 0; i < n; i++) + r[i] = (a[i] + b[i] + 1) >> 1; +} + +v8hu floor_v8hu (v8hu a, v8hu b) { return (a + b) >> 1; } + +/* { dg-final { scan-assembler {\tsrhadd\tv[0-9]+\.4s,} } } */ +/* { dg-final { scan-assembler {\tshadd\tv[0-9]+\.4s,} } } */ +/* { dg-final { scan-assembler-times {\tsrhadd\tv[0-9]+\.8h,} 1 } } */ +/* { dg-final { scan-assembler-times {\tshadd\tv[0-9]+\.8h,} 1 } } */ +/* { dg-final { scan-assembler-times {\tsrhadd\tv[0-9]+\.16b,} 1 } } */ +/* { dg-final { scan-assembler-times {\tshadd\tv[0-9]+\.16b,} 1 } } */ +/* { dg-final { scan-assembler-not {\tu(r)?hadd\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_2.c b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_2.c new file mode 100644 index 000000000000..441ffdfb8590 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/vect_hadd_same_width_2.c @@ -0,0 +1,7 @@ +/* With -fwrapv the sums can wrap, so none of these are averages. */ +/* { dg-do compile } */ +/* { dg-options "-O3 -fwrapv" } */ + +#include "vect_hadd_same_width_1.c" + +/* { dg-final { scan-assembler-not {\t[su](r)?hadd\t} } } */
