On Mon, 28 Sep 2026, Dominic P wrote: > vect_widened_op_tree accepts an operand as wide as the result only if it > is itself a promotion, so a loop invariant that merely has a narrow range > blocks every widening recogniser. x265's angular intra prediction > (source/common/intrapred.cpp:186) is the motivating case: > > f = angleSum & 31; > dst[x] = ((32 - f) * ref[x] + f * ref[x + 1] + 16) >> 5; > > Query the range of such an operand at the statement, which also picks up > a guard before the loop, and use the narrowest element type that holds it. > Only external defs are narrowed: their demoting cast is placed outside the > loop, while for an internal def such as an induction variable it would run > every iteration. The narrowed type keeps the operand's sign, so an int > invariant in [0, 31] against unsigned short data is still not widened. > > On the kernel above aarch64 -O2 goes from 76 to 67 instructions > (umull/umlal replace mul/mla/zip). > > Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression > tested on aarch64-linux-gnu (qemu).
OK. Thanks, Richard. > Assisted-by: Claude Opus 5 (Anthropic) > > gcc/ChangeLog: > > * tree-vect-patterns.cc (vect_narrow_unprom_to_range): New function. > (vect_widened_op_tree): Use it for external defs. > > gcc/testsuite/ChangeLog: > > * gcc.dg/vect/vect-widen-invariant-1.c: New test. > * gcc.dg/vect/vect-widen-invariant-2.c: New test. > * gcc.target/aarch64/widen-invariant-1.c: New test. > > Signed-off-by: Dominic P <[email protected]> > --- > Changes in v2: range_of_expr at the statement instead of > vect_get_range_info, the vect_external_def check moved to the caller, > and a test for a range that comes from a guard before the loop. > > OK for trunk? > > .../gcc.dg/vect/vect-widen-invariant-1.c | 58 +++++++++++++++++++ > .../gcc.dg/vect/vect-widen-invariant-2.c | 39 +++++++++++++ > .../gcc.target/aarch64/widen-invariant-1.c | 34 +++++++++++ > gcc/tree-vect-patterns.cc | 26 ++++++++- > 4 files changed, 156 insertions(+), 1 deletion(-) > create mode 100644 gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c > create mode 100644 gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c > create mode 100644 gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c > > diff --git a/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c > b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c > new file mode 100644 > index 00000000000..1cf9f2a97ce > --- /dev/null > +++ b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c > @@ -0,0 +1,58 @@ > +/* { dg-require-effective-target vect_int } */ > + > +#include "tree-vect.h" > + > +#define N 64 > + > +unsigned char dst[N], src[N + 1]; > +short sub[N]; > + > +__attribute__ ((noipa)) void > +ang_row (unsigned char *__restrict d, const unsigned char *__restrict s, > + int angle, int n) > +{ > + int f = angle & 31; > + int a = 32 - f; > + for (int x = 0; x < n; x++) > + d[x] = (unsigned char) ((a * s[x] + f * s[x + 1] + 16) >> 5); > +} > + > +__attribute__ ((noipa)) void > +sub_row (short *__restrict d, const unsigned char *__restrict s, > + int angle, int n) > +{ > + int f = angle & 31; > + /* Walk D with a pointer: indexing it would scale the index by 2 and give > + the test a widening multiply that has nothing to do with the patch. */ > + for (const unsigned char *e = s + n; s < e; s++) > + *d++ = (short) (f - *s); > +} > + > +int > +main (void) > +{ > + int i; > + > + check_vect (); > + > +#pragma GCC novector > + for (i = 0; i < N + 1; i++) > + src[i] = (i * 37 + 11) & 0xff; > + > + ang_row (dst, src, 0x123, N); > + sub_row (sub, src, 0x123, N); > + > +#pragma GCC novector > + for (i = 0; i < N; i++) > + { > + int f = 0x123 & 31, a = 32 - f; > + if (dst[i] != (unsigned char) ((a * src[i] + f * src[i + 1] + 16) >> > 5)) > + abort (); > + if (sub[i] != (short) (f - src[i])) > + abort (); > + } > + return 0; > +} > + > +/* { dg-final { scan-tree-dump "vect_recog_widen_mult_pattern: detected" > "vect" { target vect_widen_mult_qi_to_hi } } } */ > +/* { dg-final { scan-tree-dump "vect_recog_widen_minus_pattern: detected" > "vect" { target vect_widen_mult_qi_to_hi } } } */ > diff --git a/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c > b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c > new file mode 100644 > index 00000000000..8e47407a753 > --- /dev/null > +++ b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c > @@ -0,0 +1,39 @@ > +/* { dg-require-effective-target vect_int } */ > + > +#include "tree-vect.h" > + > +#define N 64 > + > +short src[N]; > +int dst[N]; > + > +__attribute__ ((noipa)) void > +f (int *__restrict d, const short *__restrict s, int c, int n) > +{ > + if (c < -100 || c > 100) > + return; > + for (const short *e = s + n; s < e; s++) > + *d++ = c * *s; > +} > + > +int > +main (void) > +{ > + int i; > + > + check_vect (); > + > +#pragma GCC novector > + for (i = 0; i < N; i++) > + src[i] = i * 1031 - 30000; > + > + f (dst, src, -99, N); > + > +#pragma GCC novector > + for (i = 0; i < N; i++) > + if (dst[i] != -99 * src[i]) > + abort (); > + return 0; > +} > + > +/* { dg-final { scan-tree-dump "vect_recog_widen_mult_pattern: detected" > "vect" { target vect_widen_mult_hi_to_si } } } */ > diff --git a/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c > b/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c > new file mode 100644 > index 00000000000..647b491e9df > --- /dev/null > +++ b/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c > @@ -0,0 +1,34 @@ > +/* { dg-do compile } */ > +/* { dg-options "-O2 -fno-schedule-insns -fno-schedule-insns2" } */ > +/* { dg-add-options check_function_bodies } */ > +/* { dg-final { check-function-bodies "**" "" {} } } */ > + > +/* > +** ang_row: > +** and w2, w2, 31 > +** mov w[0-9]+, 32 > +** sub w[0-9]+, w[0-9]+, w2 > +** dup v[0-9]+\.16b, w[0-9]+ > +** dup v[0-9]+\.16b, w2 > +** movi v[0-9]+\.8h, 0x10 > +** mov v[0-9]+\.16b, v[0-9]+\.16b > +** ldr q[0-9]+, \[x1\] > +** ldr q[0-9]+, \[x1, 1\] > +** umlal v[0-9]+\.8h, v[0-9]+\.8b, v[0-9]+\.8b > +** umlal v[0-9]+\.8h, v[0-9]+\.8b, v[0-9]+\.8b > +** umlal2 v[0-9]+\.8h, v[0-9]+\.16b, v[0-9]+\.16b > +** umlal2 v[0-9]+\.8h, v[0-9]+\.16b, v[0-9]+\.16b > +** shrn v[0-9]+\.8b, v[0-9]+\.8h, 5 > +** shrn2 v[0-9]+\.16b, v[0-9]+\.8h, 5 > +** str q[0-9]+, \[x0\] > +** ret > +*/ > +void > +ang_row (unsigned char *__restrict d, const unsigned char *__restrict s, > + int angle) > +{ > + int f = angle & 31; > + int a = 32 - f; > + for (int x = 0; x < 16; x++) > + d[x] = (unsigned char) ((a * s[x] + f * s[x + 1] + 16) >> 5); > +} > diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc > index 06159fa6ab8..232c9620bc0 100644 > --- a/gcc/tree-vect-patterns.cc > +++ b/gcc/tree-vect-patterns.cc > @@ -567,6 +567,28 @@ vect_joust_widened_type (tree type, tree new_type, tree > *common_type) > return true; > } > > +/* If the range of UNPROM->OP at STMT fits in a narrower element type than > + UNPROM->TYPE, change UNPROM->TYPE to that type and return true. */ > + > +static bool > +vect_narrow_unprom_to_range (vect_unpromoted_value *unprom, gimple *stmt) > +{ > + int_range_max r; > + get_range_query (cfun)->range_of_expr (r, unprom->op, stmt); > + if (r.undefined_p ()) > + return false; > + > + signop sgn = TYPE_SIGN (unprom->type); > + unsigned int precision > + = vect_element_precision (MAX (wi::min_precision (r.lower_bound (), sgn), > + wi::min_precision (r.upper_bound (), sgn))); > + if (precision >= TYPE_PRECISION (unprom->type)) > + return false; > + > + unprom->type = build_nonstandard_integer_type (precision, sgn == UNSIGNED); > + return true; > +} > + > /* Check whether STMT_INFO can be viewed as a tree of integer operations > in which each node either performs CODE or WIDENED_CODE, and where > each leaf operand is narrower than the result of STMT_INFO. MAX_NOPS > @@ -659,7 +681,9 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info > stmt_info, tree_code code, > this_unprom)) > return 0; > > - if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type)) > + if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type) > + && !(this_unprom->dt == vect_external_def > + && vect_narrow_unprom_to_range (this_unprom, stmt))) > { > /* The operand isn't widened. If STMT_INFO has the code > for an unwidened operation, recursively check whether > -- Richard Biener <[email protected]> SUSE Software Solutions Germany GmbH, Frankenstrasse 146, 90461 Nuernberg, Germany; GF: Stefan Gaiser, Jochen Jaser, Abhinav Puri; (HRB 36809, AG Nuernberg)
