vect_widened_op_tree accepts an operand that is as wide as the result only
if it is itself the demotion of a narrower value, so a loop invariant that
merely has a narrow range blocks every widening recogniser.
x265's angular intra prediction is the motivating case
(source/common/intrapred.cpp:186):
f = angleSum & 31;
dst[x] = ((32 - f) * ref[x] + f * ref[x + 1] + 16) >> 5;
with ref unsigned char. over_widening narrows the chain to unsigned
short, and widen_mult then rejects the (unsigned short) f operand even
though its range is [0, 31]. Spelling f as unsigned char in the source
makes widen_mult fire at once, so only the range query is missing.
Query the range of such an operand and record the narrower type when the
range fits at most half the result precision. vect_get_range_info is the
helper the over-widening recogniser already uses; the operand is an
external def, so its definition dominates the loop and the range holds
throughout it. The narrowed type keeps the sign of the original, as
vect_joust_widened_integer does for constants, so the value is
representable in it. The recognisers check target support for the widened
operation themselves, and for loop vectorization vect_convert_input
places the demoting cast on the incoming edge, outside the loop.
Restricting this to vect_external_def is profitability, not correctness:
the range of an induction variable is equally usable, but the demoting
cast is then executed every iteration. An earlier version admitted
induction defs and narrowed the induction variable of
gcc.dg/vect/slp-widen-mult-half.c on its [0, 15] range, changing the
pattern counts there for byte-identical code.
An operand whose range is known only after over_widening has cast it to a
new unsigned type is still missed: for signed char data and a non-negative
invariant, the cast operand has no range and the unsigned narrowing that
the range would give clashes with the signed data anyway. Fixing that
needs the sign choice to be made where the operands are jousted, not here.
On the x265 kernel above, aarch64 -mcpu=cortex-a76 -O2 goes from 77
instructions to 68: 3 mul + 2 mla + 2 zip1 + 2 zip2 become umull + umull2
+ umlal + umlal2 + 1 mul. x86_64 -O2 -mavx2 goes from 119 to 123, the
five extra instructions all in the inner loop's preheader, because x86 has
no unsigned char widening multiply and emulates it; that is what GCC
already emits there when the multiplier is spelled unsigned char in the
source, so the change makes the two spellings agree.
Bootstrapped and regression tested on x86_64-pc-linux-gnu.
Assisted-by: Claude Opus 5 (Anthropic)
gcc/ChangeLog:
* tree-vect-patterns.cc (vect_narrow_invariant_to_range): New
function.
(vect_widened_op_tree): Use it.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-widen-invariant-1.c: New test.
* gcc.target/aarch64/widen-invariant-1.c: New test.
Signed-off-by: Dominic P <[email protected]>
---
.../gcc.dg/vect/vect-widen-invariant-1.c | 58 +++++++++++++++++++
.../gcc.target/aarch64/widen-invariant-1.c | 33 +++++++++++
gcc/tree-vect-patterns.cc | 28 ++++++++-
3 files changed, 118 insertions(+), 1 deletion(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c
b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c
new file mode 100644
index 00000000000..1cf9f2a97ce
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-widen-invariant-1.c
@@ -0,0 +1,58 @@
+/* { dg-require-effective-target vect_int } */
+
+#include "tree-vect.h"
+
+#define N 64
+
+unsigned char dst[N], src[N + 1];
+short sub[N];
+
+__attribute__ ((noipa)) void
+ang_row (unsigned char *__restrict d, const unsigned char *__restrict s,
+ int angle, int n)
+{
+ int f = angle & 31;
+ int a = 32 - f;
+ for (int x = 0; x < n; x++)
+ d[x] = (unsigned char) ((a * s[x] + f * s[x + 1] + 16) >> 5);
+}
+
+__attribute__ ((noipa)) void
+sub_row (short *__restrict d, const unsigned char *__restrict s,
+ int angle, int n)
+{
+ int f = angle & 31;
+ /* Walk D with a pointer: indexing it would scale the index by 2 and give
+ the test a widening multiply that has nothing to do with the patch. */
+ for (const unsigned char *e = s + n; s < e; s++)
+ *d++ = (short) (f - *s);
+}
+
+int
+main (void)
+{
+ int i;
+
+ check_vect ();
+
+#pragma GCC novector
+ for (i = 0; i < N + 1; i++)
+ src[i] = (i * 37 + 11) & 0xff;
+
+ ang_row (dst, src, 0x123, N);
+ sub_row (sub, src, 0x123, N);
+
+#pragma GCC novector
+ for (i = 0; i < N; i++)
+ {
+ int f = 0x123 & 31, a = 32 - f;
+ if (dst[i] != (unsigned char) ((a * src[i] + f * src[i + 1] + 16) >> 5))
+ abort ();
+ if (sub[i] != (short) (f - src[i]))
+ abort ();
+ }
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "vect_recog_widen_mult_pattern: detected"
"vect" { target vect_widen_mult_qi_to_hi } } } */
+/* { dg-final { scan-tree-dump "vect_recog_widen_minus_pattern: detected"
"vect" { target vect_widen_mult_qi_to_hi } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c
b/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c
new file mode 100644
index 00000000000..24d413d71fb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/widen-invariant-1.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-schedule-insns -fno-schedule-insns2" } */
+/* { dg-final { check-function-bodies "**" "" {} } } */
+
+/*
+** ang_row:
+** and w2, w2, 31
+** mov w[0-9]+, 32
+** sub w[0-9]+, w[0-9]+, w2
+** dup v[0-9]+\.16b, w[0-9]+
+** dup v[0-9]+\.16b, w2
+** movi v[0-9]+\.8h, 0x10
+** mov v[0-9]+\.16b, v[0-9]+\.16b
+** ldr q[0-9]+, \[x1\]
+** ldr q[0-9]+, \[x1, 1\]
+** umlal v[0-9]+\.8h, v[0-9]+\.8b, v[0-9]+\.8b
+** umlal v[0-9]+\.8h, v[0-9]+\.8b, v[0-9]+\.8b
+** umlal2 v[0-9]+\.8h, v[0-9]+\.16b, v[0-9]+\.16b
+** umlal2 v[0-9]+\.8h, v[0-9]+\.16b, v[0-9]+\.16b
+** shrn v[0-9]+\.8b, v[0-9]+\.8h, 5
+** shrn2 v[0-9]+\.16b, v[0-9]+\.8h, 5
+** str q[0-9]+, \[x0\]
+** ret
+*/
+void
+ang_row (unsigned char *__restrict d, const unsigned char *__restrict s,
+ int angle)
+{
+ int f = angle & 31;
+ int a = 32 - f;
+ for (int x = 0; x < 16; x++)
+ d[x] = (unsigned char) ((a * s[x] + f * s[x + 1] + 16) >> 5);
+}
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 58710b2a5fd..98fc1be8a54 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -567,6 +567,31 @@ vect_joust_widened_type (tree type, tree new_type, tree
*common_type)
return true;
}
+/* UNPROM describes an operand of an operation whose result has type TYPE,
+ and the operand is as wide as TYPE. If the operand is loop-invariant and
+ its value range fits a type of at most half the precision of TYPE, narrow
+ UNPROM->TYPE to such a type and return true. */
+
+static bool
+vect_narrow_invariant_to_range (vect_unpromoted_value *unprom, tree type)
+{
+ wide_int min_value, max_value;
+ if (unprom->dt != vect_external_def
+ || TREE_CODE (unprom->op) != SSA_NAME
+ || !vect_get_range_info (unprom->op, &min_value, &max_value))
+ return false;
+
+ signop sgn = TYPE_SIGN (unprom->type);
+ unsigned int precision = MAX (wi::min_precision (min_value, sgn),
+ wi::min_precision (max_value, sgn));
+ if (precision * 2 > TYPE_PRECISION (type))
+ return false;
+
+ precision = vect_element_precision (precision);
+ unprom->type = build_nonstandard_integer_type (precision, sgn == UNSIGNED);
+ return true;
+}
+
/* Check whether STMT_INFO can be viewed as a tree of integer operations
in which each node either performs CODE or WIDENED_CODE, and where
each leaf operand is narrower than the result of STMT_INFO. MAX_NOPS
@@ -659,7 +684,8 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info
stmt_info, tree_code code,
this_unprom))
return 0;
- if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type))
+ if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type)
+ && !vect_narrow_invariant_to_range (this_unprom, type))
{
/* The operand isn't widened. If STMT_INFO has the code
for an unwidened operation, recursively check whether
--
2.55.0
