For a strided grouped load, get_load_store_type can drop a permutation
that selects a consecutive run of lanes and treat the run as the group,
so VMAT_STRIDED_SLP loads it with a few wide loads instead of one scalar
load per element. has_consecutive_load_permutation also required the
run length to divide the group size. Once the subchain is taken the
group size becomes the run length, and vectorizable_load's offset walk
and gcd (group_size, nunits) depend only on that, so the requirement is
not needed.
That requirement kept half-pel motion estimation,
sum += abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1)), scalar. p2[0..8]
is a group of nine, so both runs of eight were refused; the loads became
VMAT_ELEMENTWISE and the loop was not profitable. On aarch64 the kernel
now vectorizes to urhadd and uabd, 17 instructions for two rows against
62 per row.
Bootstrapped and regression tested on x86_64-pc-linux-gnu; the new
aarch64 test checked under qemu.
Assisted-by: Claude Opus 5 (Anthropic)
gcc/ChangeLog:
* tree-vect-stmts.cc (has_consecutive_load_permutation): Remove
the group_size parameter and the requirement that the permutation
length divides the group size. Make static.
(get_load_store_type): Adjust caller.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-reduc-sad-10.c: New test.
* gcc.target/aarch64/sad-halfpel-1.c: New test.
Signed-off-by: Dominic P <[email protected]>
---
gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c | 60 +++++++++++++++++++
.../gcc.target/aarch64/sad-halfpel-1.c | 20 +++++++
gcc/tree-vect-stmts.cc | 14 ++---
3 files changed, 86 insertions(+), 8 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c
b/gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c
new file mode 100644
index 00000000000..166a28012cd
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c
@@ -0,0 +1,60 @@
+/* { dg-require-effective-target vect_usad_char } */
+
+#include "tree-vect.h"
+
+#define H 16
+#define STRIDE 11
+
+unsigned char a[H * STRIDE + 1];
+unsigned char b[H * STRIDE + 1];
+
+/* Half-pel motion estimation as found in media codecs: the second SAD
+ operand averages two overlapping runs of one nine-element group. */
+
+#define TERM(K) \
+ s += __builtin_abs (p1[K] - ((p2[K] + p2[K + 1] + 1) >> 1))
+
+int __attribute__ ((noipa))
+f (unsigned char *p1, unsigned char *p2, int stride, int h)
+{
+ int s = 0;
+ for (int i = 0; i < h; i++)
+ {
+ TERM (0); TERM (1); TERM (2); TERM (3);
+ TERM (4); TERM (5); TERM (6); TERM (7);
+ p1 += stride;
+ p2 += stride;
+ }
+ return s;
+}
+
+int
+main (void)
+{
+ check_vect ();
+
+#pragma GCC novector
+ for (int i = 0; i < H * STRIDE + 1; i++)
+ {
+ a[i] = (i * 17 + 3) & 0xff;
+ b[i] = (i * 251 + 91) & 0xff;
+ }
+
+ int expected = 0;
+#pragma GCC novector
+ for (int i = 0; i < H; i++)
+ {
+ unsigned char *p1 = a + i * STRIDE;
+ unsigned char *p2 = b + i * STRIDE;
+#pragma GCC novector
+ for (int k = 0; k < 8; k++)
+ expected += __builtin_abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1));
+ }
+
+ if (f (a, b, STRIDE, H) != expected)
+ __builtin_abort ();
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "vect_recog_sad_pattern: detected" "vect" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c
b/gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c
new file mode 100644
index 00000000000..e4473653cf6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
+
+int
+pix_abs8_x2 (const unsigned char *p1, const unsigned char *p2,
+ long stride, int h)
+{
+ int s = 0;
+ for (int i = 0; i < h; i++)
+ {
+ for (int k = 0; k < 8; k++)
+ s += __builtin_abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1));
+ p1 += stride;
+ p2 += stride;
+ }
+ return s;
+}
+
+/* { dg-final { scan-assembler {\turhadd\t} } } */
+/* { dg-final { scan-assembler {\tuabd} } } */
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 09e45d0629a..65d82d51310 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2034,20 +2034,18 @@ vector_vector_composition_type (tree vtype, poly_uint64
nelts, tree *ptype,
}
/* Check if the load permutation of NODE only refers to a consecutive
- subset of the group indices where GROUP_SIZE is the size of the
- dataref's group. We also assert that the length of the permutation
- divides the group size and is a power of two.
+ subset of the group indices. We also require the length of the
+ permutation to be a power of two.
Such load permutations can be elided in strided access schemes as
we can "jump over" the gap they leave. */
-bool
-has_consecutive_load_permutation (slp_tree node, unsigned group_size)
+static bool
+has_consecutive_load_permutation (slp_tree node)
{
load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
if (!perm.exists ()
|| perm.length () <= 1
- || !pow2p_hwi (perm.length ())
- || group_size % perm.length ())
+ || !pow2p_hwi (perm.length ()))
return false;
return vect_load_perm_consecutive_p (node);
@@ -2159,7 +2157,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
/* If the load permutation is consecutive we can reduce the group to
the elements the permutation accesses. Then we release the
permutation. */
- if (has_consecutive_load_permutation (slp_node, group_size))
+ if (has_consecutive_load_permutation (slp_node))
{
ls->subchain_p = true;
group_size = SLP_TREE_LANES (slp_node);
--
2.55.0
