https://gcc.gnu.org/g:912c701af37fdc065f0a2957acd42e6b4a8e350d
commit r17-4180-g912c701af37fdc065f0a2957acd42e6b4a8e350d Author: Dominic P <[email protected]> Date: Fri Sep 11 15:59:49 2026 -0600 [PATCH v2] vect: elide a consecutive load permutation from a non-dividing group For a strided grouped load, get_load_store_type can drop a permutation that selects a consecutive run of lanes and treat the run as the group, so VMAT_STRIDED_SLP loads it with a few wide loads instead of one scalar load per element. has_consecutive_load_permutation also required the run length to divide the group size. Once the subchain is taken the group size becomes the run length, and vectorizable_load's offset walk and gcd (group_size, nunits) depend only on that, so the requirement is not needed. That requirement kept half-pel motion estimation, sum += abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1)), scalar. p2[0..8] is a group of nine, so both runs of eight were refused; the loads became VMAT_ELEMENTWISE and the loop was not profitable. On aarch64 the kernel now vectorizes to urhadd and uabd, 17 instructions for two rows against 62 per row. Bootstrapped and regression tested on x86_64-pc-linux-gnu; the new aarch64 test checked under qemu. Assisted-by: Claude Opus 5 (Anthropic) gcc/ChangeLog: * tree-vect-stmts.cc (has_consecutive_load_permutation): Remove the group_size parameter and the requirement that the permutation length divides the group size. Make static. (get_load_store_type): Adjust caller. gcc/testsuite/ChangeLog: * gcc.dg/vect/vect-reduc-sad-10.c: New test. * gcc.target/aarch64/sad-halfpel-1.c: New test. Signed-off-by: Dominic P <[email protected]> Diff: --- gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c | 60 ++++++++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c | 20 ++++++++ gcc/testsuite/gcc.target/aarch64/sad-halfpel-2.c | 27 +++++++++++ gcc/tree-vect-stmts.cc | 14 +++--- 4 files changed, 113 insertions(+), 8 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c new file mode 100644 index 000000000000..166a28012cd5 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-sad-10.c @@ -0,0 +1,60 @@ +/* { dg-require-effective-target vect_usad_char } */ + +#include "tree-vect.h" + +#define H 16 +#define STRIDE 11 + +unsigned char a[H * STRIDE + 1]; +unsigned char b[H * STRIDE + 1]; + +/* Half-pel motion estimation as found in media codecs: the second SAD + operand averages two overlapping runs of one nine-element group. */ + +#define TERM(K) \ + s += __builtin_abs (p1[K] - ((p2[K] + p2[K + 1] + 1) >> 1)) + +int __attribute__ ((noipa)) +f (unsigned char *p1, unsigned char *p2, int stride, int h) +{ + int s = 0; + for (int i = 0; i < h; i++) + { + TERM (0); TERM (1); TERM (2); TERM (3); + TERM (4); TERM (5); TERM (6); TERM (7); + p1 += stride; + p2 += stride; + } + return s; +} + +int +main (void) +{ + check_vect (); + +#pragma GCC novector + for (int i = 0; i < H * STRIDE + 1; i++) + { + a[i] = (i * 17 + 3) & 0xff; + b[i] = (i * 251 + 91) & 0xff; + } + + int expected = 0; +#pragma GCC novector + for (int i = 0; i < H; i++) + { + unsigned char *p1 = a + i * STRIDE; + unsigned char *p2 = b + i * STRIDE; +#pragma GCC novector + for (int k = 0; k < 8; k++) + expected += __builtin_abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1)); + } + + if (f (a, b, STRIDE, H) != expected) + __builtin_abort (); + return 0; +} + +/* { dg-final { scan-tree-dump "vect_recog_sad_pattern: detected" "vect" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c b/gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c new file mode 100644 index 000000000000..e4473653cf67 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sad-halfpel-1.c @@ -0,0 +1,20 @@ +/* { dg-do compile } */ +/* { dg-options "-O3" } */ + +int +pix_abs8_x2 (const unsigned char *p1, const unsigned char *p2, + long stride, int h) +{ + int s = 0; + for (int i = 0; i < h; i++) + { + for (int k = 0; k < 8; k++) + s += __builtin_abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1)); + p1 += stride; + p2 += stride; + } + return s; +} + +/* { dg-final { scan-assembler {\turhadd\t} } } */ +/* { dg-final { scan-assembler {\tuabd} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sad-halfpel-2.c b/gcc/testsuite/gcc.target/aarch64/sad-halfpel-2.c new file mode 100644 index 000000000000..55ebf33fdcdc --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sad-halfpel-2.c @@ -0,0 +1,27 @@ +/* { dg-do compile } */ +/* { dg-options "-O3" } */ + +/* The half-pel kernel of sad-halfpel-1.c, checked for how the operands are + loaded rather than for the arithmetic: the nine-element group p2[0..8] is + read by two eight-lane runs, 0..7 and 1..8, neither of which needs a load + permutation, so both must be loaded at their natural width - no + per-element loads and no vector composition. */ + +int +pix_abs8_x2 (const unsigned char *p1, const unsigned char *p2, + long stride, int h) +{ + int s = 0; + for (int i = 0; i < h; i++) + { + for (int k = 0; k < 8; k++) + s += __builtin_abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1)); + p1 += stride; + p2 += stride; + } + return s; +} + +/* { dg-final { scan-assembler {\turhadd\tv[0-9]+\.8b} } } */ +/* { dg-final { scan-assembler-not {\tldrb\t} } } */ +/* { dg-final { scan-assembler-not {\tzip1\t} } } */ diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index 09e45d0629a3..65d82d51310b 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -2034,20 +2034,18 @@ vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype, } /* Check if the load permutation of NODE only refers to a consecutive - subset of the group indices where GROUP_SIZE is the size of the - dataref's group. We also assert that the length of the permutation - divides the group size and is a power of two. + subset of the group indices. We also require the length of the + permutation to be a power of two. Such load permutations can be elided in strided access schemes as we can "jump over" the gap they leave. */ -bool -has_consecutive_load_permutation (slp_tree node, unsigned group_size) +static bool +has_consecutive_load_permutation (slp_tree node) { load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node); if (!perm.exists () || perm.length () <= 1 - || !pow2p_hwi (perm.length ()) - || group_size % perm.length ()) + || !pow2p_hwi (perm.length ())) return false; return vect_load_perm_consecutive_p (node); @@ -2159,7 +2157,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, /* If the load permutation is consecutive we can reduce the group to the elements the permutation accesses. Then we release the permutation. */ - if (has_consecutive_load_permutation (slp_node, group_size)) + if (has_consecutive_load_permutation (slp_node)) { ls->subchain_p = true; group_size = SLP_TREE_LANES (slp_node);
