https://gcc.gnu.org/g:9edd8aa65a3c2fb813412971c00981d6d466ab0f
commit r17-4801-g9edd8aa65a3c2fb813412971c00981d6d466ab0f Author: Dominic P <[email protected]> Date: Mon Sep 28 12:05:51 2026 +0100 vect: Use the range of a loop-invariant operand when widening vect_widened_op_tree accepts an operand as wide as the result only if it is itself a promotion, so a loop invariant that merely has a narrow range blocks every widening recogniser. x265's angular intra prediction (source/common/intrapred.cpp:186) is the motivating case: f = angleSum & 31; dst[x] = ((32 - f) * ref[x] + f * ref[x + 1] + 16) >> 5; Query the range of such an operand at the statement, which also picks up a guard before the loop, and use the narrowest element type that holds it. Only external defs are narrowed: their demoting cast is placed outside the loop, while for an internal def such as an induction variable it would run every iteration. The narrowed type keeps the operand's sign, so an int invariant in [0, 31] against unsigned short data is still not widened. On the kernel above aarch64 -O2 goes from 76 to 67 instructions (umull/umlal replace mul/mla/zip). Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression tested on aarch64-linux-gnu (qemu). Assisted-by: Claude Opus 5 (Anthropic) gcc/ChangeLog: * tree-vect-patterns.cc (vect_narrow_unprom_to_range): New function. (vect_widened_op_tree): Use it for external defs. gcc/testsuite/ChangeLog: * gcc.dg/vect/vect-widen-invariant-1.c: New test. * gcc.dg/vect/vect-widen-invariant-2.c: New test. * gcc.target/aarch64/widen-invariant-1.c: New test. Signed-off-by: Dominic P <[email protected]> Diff: --- gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c | 58 ++++++++++++++++++++++ gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c | 39 +++++++++++++++ .../gcc.target/aarch64/widen-invariant-1.c | 34 +++++++++++++ gcc/tree-vect-patterns.cc | 26 +++++++++- 4 files changed, 156 insertions(+), 1 deletion(-) diff --git a/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c new file mode 100644 index 000000000000..1cf9f2a97ce7 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c @@ -0,0 +1,58 @@ +/* { dg-require-effective-target vect_int } */ + +#include "tree-vect.h" + +#define N 64 + +unsigned char dst[N], src[N + 1]; +short sub[N]; + +__attribute__ ((noipa)) void +ang_row (unsigned char *__restrict d, const unsigned char *__restrict s, + int angle, int n) +{ + int f = angle & 31; + int a = 32 - f; + for (int x = 0; x < n; x++) + d[x] = (unsigned char) ((a * s[x] + f * s[x + 1] + 16) >> 5); +} + +__attribute__ ((noipa)) void +sub_row (short *__restrict d, const unsigned char *__restrict s, + int angle, int n) +{ + int f = angle & 31; + /* Walk D with a pointer: indexing it would scale the index by 2 and give + the test a widening multiply that has nothing to do with the patch. */ + for (const unsigned char *e = s + n; s < e; s++) + *d++ = (short) (f - *s); +} + +int +main (void) +{ + int i; + + check_vect (); + +#pragma GCC novector + for (i = 0; i < N + 1; i++) + src[i] = (i * 37 + 11) & 0xff; + + ang_row (dst, src, 0x123, N); + sub_row (sub, src, 0x123, N); + +#pragma GCC novector + for (i = 0; i < N; i++) + { + int f = 0x123 & 31, a = 32 - f; + if (dst[i] != (unsigned char) ((a * src[i] + f * src[i + 1] + 16) >> 5)) + abort (); + if (sub[i] != (short) (f - src[i])) + abort (); + } + return 0; +} + +/* { dg-final { scan-tree-dump "vect_recog_widen_mult_pattern: detected" "vect" { target vect_widen_mult_qi_to_hi } } } */ +/* { dg-final { scan-tree-dump "vect_recog_widen_minus_pattern: detected" "vect" { target vect_widen_mult_qi_to_hi } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c new file mode 100644 index 000000000000..8e47407a7534 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-2.c @@ -0,0 +1,39 @@ +/* { dg-require-effective-target vect_int } */ + +#include "tree-vect.h" + +#define N 64 + +short src[N]; +int dst[N]; + +__attribute__ ((noipa)) void +f (int *__restrict d, const short *__restrict s, int c, int n) +{ + if (c < -100 || c > 100) + return; + for (const short *e = s + n; s < e; s++) + *d++ = c * *s; +} + +int +main (void) +{ + int i; + + check_vect (); + +#pragma GCC novector + for (i = 0; i < N; i++) + src[i] = i * 1031 - 30000; + + f (dst, src, -99, N); + +#pragma GCC novector + for (i = 0; i < N; i++) + if (dst[i] != -99 * src[i]) + abort (); + return 0; +} + +/* { dg-final { scan-tree-dump "vect_recog_widen_mult_pattern: detected" "vect" { target vect_widen_mult_hi_to_si } } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c b/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c new file mode 100644 index 000000000000..647b491e9df7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c @@ -0,0 +1,34 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fno-schedule-insns -fno-schedule-insns2" } */ +/* { dg-add-options check_function_bodies } */ +/* { dg-final { check-function-bodies "**" "" {} } } */ + +/* +** ang_row: +** and w2, w2, 31 +** mov w[0-9]+, 32 +** sub w[0-9]+, w[0-9]+, w2 +** dup v[0-9]+\.16b, w[0-9]+ +** dup v[0-9]+\.16b, w2 +** movi v[0-9]+\.8h, 0x10 +** mov v[0-9]+\.16b, v[0-9]+\.16b +** ldr q[0-9]+, \[x1\] +** ldr q[0-9]+, \[x1, 1\] +** umlal v[0-9]+\.8h, v[0-9]+\.8b, v[0-9]+\.8b +** umlal v[0-9]+\.8h, v[0-9]+\.8b, v[0-9]+\.8b +** umlal2 v[0-9]+\.8h, v[0-9]+\.16b, v[0-9]+\.16b +** umlal2 v[0-9]+\.8h, v[0-9]+\.16b, v[0-9]+\.16b +** shrn v[0-9]+\.8b, v[0-9]+\.8h, 5 +** shrn2 v[0-9]+\.16b, v[0-9]+\.8h, 5 +** str q[0-9]+, \[x0\] +** ret +*/ +void +ang_row (unsigned char *__restrict d, const unsigned char *__restrict s, + int angle) +{ + int f = angle & 31; + int a = 32 - f; + for (int x = 0; x < 16; x++) + d[x] = (unsigned char) ((a * s[x] + f * s[x + 1] + 16) >> 5); +} diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc index 06159fa6ab8f..232c9620bc05 100644 --- a/gcc/tree-vect-patterns.cc +++ b/gcc/tree-vect-patterns.cc @@ -567,6 +567,28 @@ vect_joust_widened_type (tree type, tree new_type, tree *common_type) return true; } +/* If the range of UNPROM->OP at STMT fits in a narrower element type than + UNPROM->TYPE, change UNPROM->TYPE to that type and return true. */ + +static bool +vect_narrow_unprom_to_range (vect_unpromoted_value *unprom, gimple *stmt) +{ + int_range_max r; + get_range_query (cfun)->range_of_expr (r, unprom->op, stmt); + if (r.undefined_p ()) + return false; + + signop sgn = TYPE_SIGN (unprom->type); + unsigned int precision + = vect_element_precision (MAX (wi::min_precision (r.lower_bound (), sgn), + wi::min_precision (r.upper_bound (), sgn))); + if (precision >= TYPE_PRECISION (unprom->type)) + return false; + + unprom->type = build_nonstandard_integer_type (precision, sgn == UNSIGNED); + return true; +} + /* Check whether STMT_INFO can be viewed as a tree of integer operations in which each node either performs CODE or WIDENED_CODE, and where each leaf operand is narrower than the result of STMT_INFO. MAX_NOPS @@ -659,7 +681,9 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, this_unprom)) return 0; - if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type)) + if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type) + && !(this_unprom->dt == vect_external_def + && vect_narrow_unprom_to_range (this_unprom, stmt))) { /* The operand isn't widened. If STMT_INFO has the code for an unwidened operation, recursively check whether
