On Mon, 28 Sep 2026, Dominic P wrote:
> get_load_store_type elides a consecutive load permutation of a strided
> group and vectorizable_load re-derives the group from subchain_p.
> Release it in remove_redundant_permutations instead, with the BB subchain
> walk kept to power-of-two runs of two or more lanes (other runs load in
> gcd (lanes, nunits) pieces: five of eight shorts ran 1.8 times slower on
> x86_64), and let a node without a load permutation be its own group.
>
> Runs at lane 0 whose permutation is dropped at VF 1 were costed as
> element loads; the half-pel SAD loop is now 24 to 28% faster on x86_64
> for 64-row blocks. On a Cortex-A72, 16-byte rows are 0.2 to 2.2% faster
> and 8-byte rows flat or up to 4.1% faster with generic tuning; with
> -mcpu=cortex-a72 they are 3.9% faster for 8-row blocks, flat for 16 and
> 8.5% slower for 64: the cost model now picks a one-row V2SI loop.
>
> Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression
> tested on aarch64-linux-gnu under qemu.
>
> Assisted-by: Claude Opus 5 (Anthropic)
>
> gcc/ChangeLog:
>
> * tree-vect-slp.cc
> (vect_optimize_slp_pass::remove_redundant_permutations): Release
> the load permutation of a consecutive subchain of a strided group
> with a power-of-two number of lanes greater than one.
> * tree-vect-stmts.cc (has_consecutive_load_permutation): Remove.
> (get_load_store_type): Do not release the load permutation here.
> Take the group size from the SLP node when the node has none.
> (vectorizable_load): Take the group size and the first statement
> of the group from the SLP node when the node has no load
> permutation.
> * tree-vectorizer.h (vect_load_store_data): Remove subchain_p.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.dg/vect/vect-strided-subrun-1.c: New test.
>
> Signed-off-by: Dominic P <[email protected]>
> ---
> Changes in v3: the BB subchain walk replaces v2's own copy of it, as you
> suggested. With -mcpu=neoverse-v2, neoverse-n2 or cortex-a710 the 8-byte
> loop is now unrolled twice and so gets no vector epilogue, as a plain
> 8-byte SAD already does; odd heights run one row as scalar code.
>
> OK for trunk?
>
> .../gcc.dg/vect/vect-strided-subrun-1.c | 137 ++++++++++++++++++
> gcc/tree-vect-slp.cc | 13 +-
> gcc/tree-vect-stmts.cc | 67 ++-------
> gcc/tree-vectorizer.h | 2 -
> 4 files changed, 161 insertions(+), 58 deletions(-)
> create mode 100644 gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
>
> diff --git a/gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
> b/gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
> new file mode 100644
> index 00000000000..7b2bf6b2247
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
> @@ -0,0 +1,137 @@
> +/* { dg-require-effective-target vect_int } */
> +
> +/* Consecutive runs of a strided load group that start at its first
> + element or later, cover part of it, have an odd length or a single
> + lane, overlap, run at a negative stride or feed two consumers. */
> +
> +#include "tree-vect.h"
> +
> +#define N 4096
> +unsigned char buf[N];
> +
> +#define KERNELS(NAME, ATTR) \
> +ATTR int \
> +NAME##_halfpel (const unsigned char *p1, const unsigned char *p2, \
> + long stride, int h) \
> +{ \
> + int s = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 8; k++) \
> + s += __builtin_abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1)); \
> + p1 += stride; \
> + p2 += stride; \
> + }
> \
> + return s; \
> +} \
> + \
> +ATTR int \
> +NAME##_run_at_5 (const unsigned char *p, long stride, int h) \
> +{ \
> + int s = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 4; k++) \
> + s += p[k + 5] * 3 + p[k + 8]; \
> + s += p[0] + p[11]; \
> + p += stride; \
> + }
> \
> + return s; \
> +} \
> + \
> +ATTR int \
> +NAME##_negstep (const unsigned char *p, long stride, int h) \
> +{ \
> + int s = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 8; k++) \
> + s += __builtin_abs (p[k] - ((p[k + 1] + 1) >> 1)); \
> + p -= stride; \
> + }
> \
> + return s; \
> +} \
> + \
> +ATTR int \
> +NAME##_shared (const unsigned char *p, long stride, int h) \
> +{ \
> + int s = 0, t = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 8; k++) \
> + { \
> + int v = p[k + 1]; \
> + s += v * 3; \
> + t += v ^ p[k]; \
> + } \
> + p += stride; \
> + }
> \
> + return s - t;
> \
> +} \
> + \
> +ATTR int \
> +NAME##_lane0 (const unsigned char *p, long stride, int h) \
> +{ \
> + int s = 0, t = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 4; k++) \
> + { \
> + s += p[k] * 3; \
> + t ^= p[k + 4]; \
> + } \
> + p += stride; \
> + }
> \
> + return s - t;
> \
> +} \
> + \
> +ATTR int \
> +NAME##_odd (const unsigned char *p, long stride, int h)
> \
> +{ \
> + int s = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 3; k++) \
> + s += p[k + 1] * 5; \
> + s += p[0] + p[5] * 7; \
> + p += stride; \
> + }
> \
> + return s; \
> +}
> +
> +KERNELS (vec, __attribute__ ((noipa)))
> +KERNELS (ref, __attribute__ ((noipa, optimize ("O0"))))
> +
> +int
> +main (void)
> +{
> + check_vect ();
> +
> + unsigned x = 1;
> + for (int i = 0; i < N; i++)
> + {
> + x = x * 1103515245u + 12345u;
> + buf[i] = x >> 16;
> + }
> +
> + for (long stride = 13; stride <= 40; stride += 27)
> + for (int h = 1; h <= 33; h += 8)
> + {
> + if (vec_halfpel (buf + 3, buf + 700, stride, h)
> + != ref_halfpel (buf + 3, buf + 700, stride, h))
> + __builtin_abort ();
> + if (vec_run_at_5 (buf + 1, stride, h)
> + != ref_run_at_5 (buf + 1, stride, h))
> + __builtin_abort ();
> + if (vec_negstep (buf + 2000, stride, h)
> + != ref_negstep (buf + 2000, stride, h))
> + __builtin_abort ();
> + if (vec_shared (buf + 5, stride, h) != ref_shared (buf + 5, stride, h))
> + __builtin_abort ();
> + if (vec_lane0 (buf + 7, stride, h) != ref_lane0 (buf + 7, stride, h))
> + __builtin_abort ();
> + if (vec_odd (buf + 9, stride, h) != ref_odd (buf + 9, stride, h))
> + __builtin_abort ();
> + }
> + return 0;
> +}
> diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
> index 12b1abe3d78..8ca291d2bd3 100644
> --- a/gcc/tree-vect-slp.cc
> +++ b/gcc/tree-vect-slp.cc
> @@ -8161,10 +8161,15 @@ vect_optimize_slp_pass::remove_redundant_permutations
> ()
> if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
> continue;
>
> - /* In basic block vectorization we allow any subchain of an
> interleaving
> - chain.
> - FORNOW: not in loop SLP because of realignment complications. */
> - if (is_a <bb_vec_info> (m_vinfo))
> + /* In basic block vectorization we allow any subchain of an
> + interleaving chain, for strided loop accesses a power-of-two run
> + of two or more lanes, which is loaded without splitting.
> + FORNOW: not otherwise in loop SLP because of realignment
> + complications. */
> + if (is_a <bb_vec_info> (m_vinfo)
> + || (STMT_VINFO_STRIDED_P (SLP_TREE_SCALAR_STMTS (node)[0])
> + && SLP_TREE_LANES (node) > 1
> + && pow2p_hwi (SLP_TREE_LANES (node))))
My question from the previous version still stands - what is special
about pow2p_hwi number of lanes? Having test coverage for the
non-power-of-two case would be nice, of course. We're probably
filling the vector elementwise (there's room for improvement
in vectorizable_load), but given the whole DR group size can
be non-power-of-two size as well I don't see how this is a
reliable cost thing? Please at least add a comment here.
I _think_ cost-wide we want to optimize n in
if (memory_access_type == VMAT_STRIDED_SLP)
{
HOST_WIDE_INT n = gcd (group_size, const_nunits);
so n for DR_GROUP_SIZE vs n for SLP_TREE_LANES, esp. we'd like to
preserve n == const_nunits. I somewhat hate to bring in more
SLP_TREE_VECTYPE uses into permute optimizations, but well ...
we can also chose to defer this to vectorizable_load which
can still decide to load the whole group and permute.
Richard.
> {
> bool subchain_p = true;
> stmt_vec_info next_load_info = NULL;
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index 244f319d0e4..186228ba23d 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -2033,25 +2033,6 @@ vector_vector_composition_type (tree vtype,
> poly_uint64 nelts, tree *ptype,
> return NULL_TREE;
> }
>
> -/* Check if the load permutation of NODE only refers to a consecutive
> - subset of the group indices. We also require the length of the
> - permutation to be a power of two.
> - Such load permutations can be elided in strided access schemes as
> - we can "jump over" the gap they leave. */
> -
> -static bool
> -has_consecutive_load_permutation (slp_tree node)
> -{
> - load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
> - if (!perm.exists ()
> - || perm.length () <= 1
> - || !pow2p_hwi (perm.length ()))
> - return false;
> -
> - return vect_load_perm_consecutive_p (node);
> -}
> -
> -
> /* Analyze load or store SLP_NODE of type VLS_TYPE. Return true
> if there is a memory access type that the vectorized form can use,
> storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
> @@ -2101,7 +2082,6 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> *slp_perm = false;
> *n_perms = -1U;
> *n_loads = -1U;
> - ls->subchain_p = false;
>
> bool perm_ok = true;
> poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
> @@ -2143,15 +2123,10 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> Fall back to scalar accesses if that isn't possible. */
> *memory_access_type = VMAT_STRIDED_SLP;
>
> - /* If the load permutation is consecutive we can reduce the group to
> - the elements the permutation accesses. Then we release the
> - permutation. */
> - if (has_consecutive_load_permutation (slp_node))
> - {
> - ls->subchain_p = true;
> - group_size = SLP_TREE_LANES (slp_node);
> - SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
> - }
> + /* A load permutation indexes the whole group, so we have to load
> + all of it; without one the node's lanes are the group. */
> + if (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
> + group_size = SLP_TREE_LANES (slp_node);
> }
> else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> @@ -10197,16 +10172,14 @@ vectorizable_load (vec_info *vinfo,
> gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
> gcc_assert (!nested_in_vect_loop);
>
> + /* A load permutation indexes the whole group; without one the
> + node's own statements are the group. */
> + bool perm_p = SLP_TREE_LOAD_PERMUTATION (slp_node).exists ();
> if (grouped_load)
> {
> - /* If we elided a consecutive load permutation, don't
> - use the original first statement (which could be elided)
> - but the one the load permutation starts with.
> - This ensures the stride_base below is correct. */
> - if (!ls.subchain_p)
> - first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
> - else
> - first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
> + first_stmt_info = (perm_p
> + ? DR_GROUP_FIRST_ELEMENT (stmt_info)
> + : SLP_TREE_SCALAR_STMTS (slp_node)[0]);
> first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
> ref_type = get_group_alias_ptr_type (first_stmt_info);
> }
> @@ -10217,22 +10190,12 @@ vectorizable_load (vec_info *vinfo,
> ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
> }
>
> - if (grouped_load)
> - {
> - if (memory_access_type == VMAT_STRIDED_SLP)
> - {
> - /* If we elided a consecutive load permutation, adjust
> - the group size here. */
> - if (!ls.subchain_p)
> - group_size = DR_GROUP_SIZE (first_stmt_info);
> - else
> - group_size = SLP_TREE_LANES (slp_node);
> - }
> - else /* VMAT_ELEMENTWISE */
> - group_size = SLP_TREE_LANES (slp_node);
> - }
> - else
> + if (!grouped_load)
> group_size = 1;
> + else if (perm_p && memory_access_type == VMAT_STRIDED_SLP)
> + group_size = DR_GROUP_SIZE (first_stmt_info);
> + else
> + group_size = SLP_TREE_LANES (slp_node);
>
> if (!costing_p)
> {
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index abd9edc2743..26df0ad526e 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -312,8 +312,6 @@ struct vect_load_store_data : vect_data {
> bool slp_perm; // SLP_TREE_LOAD_PERMUTATION
> unsigned n_perms; // SLP_TREE_LOAD_PERMUTATION
> unsigned n_loads; // SLP_TREE_LOAD_PERMUTATION
> - /* Whether the load permutation is consecutive and simple. */
> - bool subchain_p; // VMAT_STRIDED_SLP and VMAT_GATHER_SCATTER
> };
>
> struct match_elt_t {
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Stefan Gaiser, Jochen Jaser, Abhinav Puri; (HRB 36809, AG Nuernberg)
