On Tue, 22 Sep 2026, Dominic P wrote:
> get_load_store_type elides a consecutive load permutation of a strided
> group, records that in subchain_p, and vectorizable_load re-derives the
> group from the flag. Elide it in remove_redundant_permutations instead:
> a load permutation indexes the whole group, and a node without one is its
> own group. That also covers a run that never had a permutation, which
> the flag could not see: in the half-pel SAD kernel p2[1..8] got one wide
> load while p2[0..7], starting at lane 0, kept DR_GROUP_SIZE 9 and was
> built from per-element loads. The rule in vectorizable_load is wider
> than the elision: it also covers a permutation-free node on the
> negative-constant-step route to VMAT_STRIDED_SLP.
>
> On aarch64 that loop goes from 50 to 23 instructions, and
> gcc.target/aarch64/sad-halfpel-2.c, which has failed since r17-4180,
> passes. On x86_64 the function grows, 148 to 167 instructions at -O3
> and 131 to 144 with -mavx2, and gets faster: measured natively, 8-row
> blocks are unchanged at -O3 and 14% faster with -mavx2, 64-row blocks
> 26 to 30% faster at -O3, -msse4.2 and -mavx2.
>
> Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression
> tested on aarch64-linux-gnu under qemu.
>
> Assisted-by: Claude Opus 5 (Anthropic)
>
> gcc/ChangeLog:
>
> * tree-vect-slp.cc
> (vect_optimize_slp_pass::remove_redundant_permutations): Release a
> consecutive load permutation of a strided group.
> * tree-vect-stmts.cc (has_consecutive_load_permutation): Remove.
> (get_load_store_type): Take the group size from the SLP node when
> the node has no load permutation.
> (vectorizable_load): Likewise, and take the first statement of the
> group from the node too.
> * tree-vectorizer.h (vect_load_store_data): Remove subchain_p.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.dg/vect/vect-strided-subrun-1.c: New test.
>
> Signed-off-by: Dominic P <[email protected]>
> ---
> v2: rebased on trunk 2cb78d1a0e5. The vectorizer change is unchanged from
> v1; sad-halfpel-2.c went in with r17-4180 already, which is why Linaro's CI
> reports it failing on aarch64 (GNU-2036), and this fixes that. New: a
> run-time test for sub-runs of a strided group, and x86_64 numbers measured
> again on this base (v1 quoted one block height only).
>
> .../gcc.dg/vect/vect-strided-subrun-1.c | 102 ++++++++++++++++++
> gcc/tree-vect-slp.cc | 11 ++
> gcc/tree-vect-stmts.cc | 67 +++---------
> gcc/tree-vectorizer.h | 2 -
> 4 files changed, 128 insertions(+), 54 deletions(-)
> create mode 100644 gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
>
> diff --git a/gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
> b/gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
> new file mode 100644
> index 00000000000..1882c63d75e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/vect-strided-subrun-1.c
> @@ -0,0 +1,102 @@
> +/* { dg-require-effective-target vect_int } */
> +
> +/* Consecutive runs of a strided load group that do not start at its first
> + element, overlap, run at a negative stride or feed two consumers. */
> +
> +#include "tree-vect.h"
> +
> +#define N 4096
> +unsigned char buf[N];
> +
> +#define KERNELS(NAME, ATTR) \
> +ATTR int \
> +NAME##_halfpel (const unsigned char *p1, const unsigned char *p2, \
> + long stride, int h) \
> +{ \
> + int s = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 8; k++) \
> + s += __builtin_abs (p1[k] - ((p2[k] + p2[k + 1] + 1) >> 1)); \
> + p1 += stride; \
> + p2 += stride; \
> + }
> \
> + return s; \
> +} \
> + \
> +ATTR int \
> +NAME##_run_at_5 (const unsigned char *p, long stride, int h) \
> +{ \
> + int s = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 4; k++) \
> + s += p[k + 5] * 3 + p[k + 8]; \
> + s += p[0] + p[11]; \
> + p += stride; \
> + }
> \
> + return s; \
> +} \
> + \
> +ATTR int \
> +NAME##_negstep (const unsigned char *p, long stride, int h) \
> +{ \
> + int s = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 8; k++) \
> + s += __builtin_abs (p[k] - ((p[k + 1] + 1) >> 1)); \
> + p -= stride; \
> + }
> \
> + return s; \
> +} \
> + \
> +ATTR int \
> +NAME##_shared (const unsigned char *p, long stride, int h) \
> +{ \
> + int s = 0, t = 0; \
> + for (int i = 0; i < h; i++)
> \
> + {
> \
> + for (int k = 0; k < 8; k++) \
> + { \
> + int v = p[k + 1]; \
> + s += v * 3; \
> + t += v ^ p[k]; \
> + } \
> + p += stride; \
> + }
> \
> + return s - t;
> \
> +}
> +
> +KERNELS (vec, __attribute__ ((noipa)))
> +KERNELS (ref, __attribute__ ((noipa, optimize ("O0"))))
> +
> +int
> +main (void)
> +{
> + check_vect ();
> +
> + unsigned x = 1;
> + for (int i = 0; i < N; i++)
> + {
> + x = x * 1103515245u + 12345u;
> + buf[i] = x >> 16;
> + }
> +
> + for (long stride = 13; stride <= 40; stride += 27)
> + for (int h = 1; h <= 33; h += 8)
> + {
> + if (vec_halfpel (buf + 3, buf + 700, stride, h)
> + != ref_halfpel (buf + 3, buf + 700, stride, h))
> + __builtin_abort ();
> + if (vec_run_at_5 (buf + 1, stride, h)
> + != ref_run_at_5 (buf + 1, stride, h))
> + __builtin_abort ();
> + if (vec_negstep (buf + 2000, stride, h)
> + != ref_negstep (buf + 2000, stride, h))
> + __builtin_abort ();
> + if (vec_shared (buf + 5, stride, h) != ref_shared (buf + 5, stride, h))
> + __builtin_abort ();
> + }
> + return 0;
> +}
> diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
> index 12b1abe3d78..77b5e260b44 100644
> --- a/gcc/tree-vect-slp.cc
> +++ b/gcc/tree-vect-slp.cc
> @@ -8204,6 +8204,17 @@ vect_optimize_slp_pass::remove_redundant_permutations
> ()
> }
> stmt_vec_info first_stmt_info
> = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
> + /* For a strided group the group is only a lane numbering, so a
> + consecutive run of it is itself a group with the same DR_STEP
> + and the permutation selecting the run is redundant. */
> + if (STMT_VINFO_STRIDED_P (first_stmt_info)
> + && SLP_TREE_LANES (node) > 1
> + && pow2p_hwi (SLP_TREE_LANES (node))
why this?
> + && vect_load_perm_consecutive_p (node))
I think this might go wrong when SLP_TREE_LANES != group_size, that is,
when the load permutation selects a subset of the group starting at
lane zero. For BB vectorization we allow _any_ consecutive sub-chain
(the preceeding code handles this). So I think the BB case should
be extendable to the loop case for STMT_VINFO_STRIDED_P with the
argument you rise here.
I'll note that some checks which are at the moment based on
the existence of a SLP_TREE_LOAD_PERMUTATION might be mis-using that
and could now fail to trigger.
> + {
> + SLP_TREE_LOAD_PERMUTATION (node).release ();
> + continue;
> + }
> if (!this_load_permuted
> /* The load requires permutation when unrolling exposes
> a gap either because the group is larger than the SLP
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index eb78fd2eba3..53f863cb3ed 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -2033,25 +2033,6 @@ vector_vector_composition_type (tree vtype,
> poly_uint64 nelts, tree *ptype,
> return NULL_TREE;
> }
>
> -/* Check if the load permutation of NODE only refers to a consecutive
> - subset of the group indices. We also require the length of the
> - permutation to be a power of two.
> - Such load permutations can be elided in strided access schemes as
> - we can "jump over" the gap they leave. */
> -
> -static bool
> -has_consecutive_load_permutation (slp_tree node)
> -{
> - load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
> - if (!perm.exists ()
> - || perm.length () <= 1
> - || !pow2p_hwi (perm.length ()))
> - return false;
> -
> - return vect_load_perm_consecutive_p (node);
> -}
> -
> -
> /* Analyze load or store SLP_NODE of type VLS_TYPE. Return true
> if there is a memory access type that the vectorized form can use,
> storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
> @@ -2101,7 +2082,6 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> *slp_perm = false;
> *n_perms = -1U;
> *n_loads = -1U;
> - ls->subchain_p = false;
>
> bool perm_ok = true;
> poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
> @@ -2154,15 +2134,10 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
> stmt_info,
> Fall back to scalar accesses if that isn't possible. */
> *memory_access_type = VMAT_STRIDED_SLP;
>
> - /* If the load permutation is consecutive we can reduce the group to
> - the elements the permutation accesses. Then we release the
> - permutation. */
> - if (has_consecutive_load_permutation (slp_node))
> - {
> - ls->subchain_p = true;
> - group_size = SLP_TREE_LANES (slp_node);
> - SLP_TREE_LOAD_PERMUTATION (slp_node).release ();
> - }
> + /* A load permutation indexes the whole group, so we have to load
> + all of it; without one the node's lanes are the group. */
> + if (!SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
> + group_size = SLP_TREE_LANES (slp_node);
> }
> else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> {
> @@ -10190,16 +10165,14 @@ vectorizable_load (vec_info *vinfo,
> gcc_assert (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
> gcc_assert (!nested_in_vect_loop);
>
> + /* A load permutation indexes the whole group; without one the
> + node's own statements are the group. */
> + bool perm_p = SLP_TREE_LOAD_PERMUTATION (slp_node).exists ();
> if (grouped_load)
> {
> - /* If we elided a consecutive load permutation, don't
> - use the original first statement (which could be elided)
> - but the one the load permutation starts with.
> - This ensures the stride_base below is correct. */
> - if (!ls.subchain_p)
> - first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
> - else
> - first_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[0];
> + first_stmt_info = (perm_p
> + ? DR_GROUP_FIRST_ELEMENT (stmt_info)
> + : SLP_TREE_SCALAR_STMTS (slp_node)[0]);
> first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
> ref_type = get_group_alias_ptr_type (first_stmt_info);
> }
> @@ -10210,22 +10183,12 @@ vectorizable_load (vec_info *vinfo,
> ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
> }
>
> - if (grouped_load)
> - {
> - if (memory_access_type == VMAT_STRIDED_SLP)
> - {
> - /* If we elided a consecutive load permutation, adjust
> - the group size here. */
> - if (!ls.subchain_p)
> - group_size = DR_GROUP_SIZE (first_stmt_info);
> - else
> - group_size = SLP_TREE_LANES (slp_node);
> - }
> - else /* VMAT_ELEMENTWISE */
> - group_size = SLP_TREE_LANES (slp_node);
> - }
> - else
> + if (!grouped_load)
> group_size = 1;
> + else if (perm_p && memory_access_type == VMAT_STRIDED_SLP)
> + group_size = DR_GROUP_SIZE (first_stmt_info);
> + else
> + group_size = SLP_TREE_LANES (slp_node);
So I _think_ this should work for sub-chains not starting at lane 0.
Richard.
> if (!costing_p)
> {
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index abd9edc2743..26df0ad526e 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -312,8 +312,6 @@ struct vect_load_store_data : vect_data {
> bool slp_perm; // SLP_TREE_LOAD_PERMUTATION
> unsigned n_perms; // SLP_TREE_LOAD_PERMUTATION
> unsigned n_loads; // SLP_TREE_LOAD_PERMUTATION
> - /* Whether the load permutation is consecutive and simple. */
> - bool subchain_p; // VMAT_STRIDED_SLP and VMAT_GATHER_SCATTER
> };
>
> struct match_elt_t {
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Stefan Gaiser, Jochen Jaser, Abhinav Puri; (HRB 36809, AG Nuernberg)
