https://gcc.gnu.org/g:273bff9cc9fc1a61dd8c0997ecbc12f865578f38
commit r17-5004-g273bff9cc9fc1a61dd8c0997ecbc12f865578f38 Author: Dominic P <[email protected]> Date: Mon Oct 5 20:50:08 2026 +0100 vect: Account for peeling by masking in strided gather offsets vect_truncate_gather_scatter_offset bounds the offsets { 0, X, X*2, ... } by the latch iteration count, but peeling for alignment by masking shifts the active lanes up by the number of skipped iterations. With -msve-vector-bits=256 the new tests access lane 7 through the 32-bit offset 7 * 650000000, which wraps. Do not lower the bound when peeling for alignment, since whether a mask will do the peeling is not yet known. GCC 11 onwards have the same wrong code. Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression tested on aarch64-linux-gnu (qemu, default and SVE-256 boards). Assisted-by: Claude Opus 5 (Anthropic) gcc/ChangeLog: * tree-vect-stmts.cc (vect_truncate_gather_scatter_offset): Do not lower the element count to the latch iteration count when peeling for alignment. gcc/testsuite/ChangeLog: * gcc.target/aarch64/sve/strided_load_9.c: New test. * gcc.target/aarch64/sve/strided_store_8.c: New test. Signed-off-by: Dominic P <[email protected]> Diff: --- gcc/testsuite/gcc.target/aarch64/sve/strided_load_9.c | 16 ++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/strided_store_8.c | 16 ++++++++++++++++ gcc/tree-vect-stmts.cc | 9 +++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/gcc/testsuite/gcc.target/aarch64/sve/strided_load_9.c b/gcc/testsuite/gcc.target/aarch64/sve/strided_load_9.c new file mode 100644 index 000000000000..b5e793f4394b --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/strided_load_9.c @@ -0,0 +1,16 @@ +/* { dg-do compile { target lp64 } } */ +/* { dg-options "-O3 -fno-vect-cost-model -msve-vector-bits=256 -mautovec-preference=sve-only" } */ + +int x[8] __attribute__ ((aligned (64))); + +void +foo (int *restrict b, unsigned int n) +{ + for (unsigned int i = 1; i < n; ++i) + x[i] = b[i * 650000000L]; +} + +/* Peeling for alignment by masking makes lane 7 active; 7 * 650000000 + does not fit in 32 bits. */ +/* { dg-final { scan-assembler-not {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x[0-9]+, z[0-9]+\.s, [su]xtw} } } */ +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d, p[0-7]/z, \[x[0-9]+, z[0-9]+\.d\]} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/strided_store_8.c b/gcc/testsuite/gcc.target/aarch64/sve/strided_store_8.c new file mode 100644 index 000000000000..6e6602eb923c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/strided_store_8.c @@ -0,0 +1,16 @@ +/* { dg-do compile { target lp64 } } */ +/* { dg-options "-O3 -fno-vect-cost-model -msve-vector-bits=256 -mautovec-preference=sve-only" } */ + +int x[8] __attribute__ ((aligned (64))); + +void +bar (int *restrict b, unsigned int n) +{ + for (unsigned int i = 1; i < n; ++i) + b[i * 650000000L] = x[i]; +} + +/* Peeling for alignment by masking makes lane 7 active; 7 * 650000000 + does not fit in 32 bits. */ +/* { dg-final { scan-assembler-not {\tst1w\tz[0-9]+\.s, p[0-7], \[x[0-9]+, z[0-9]+\.s, [su]xtw} } } */ +/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.d, p[0-7], \[x[0-9]+, z[0-9]+\.d\]} 2 } } */ diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index 18d0dd8ff3a3..6a2e42aac961 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -1653,10 +1653,15 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype, Start with the maximum vectorization factor. */ unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1; - /* Try lowering COUNT to the number of scalar latch iterations. */ + /* Try lowering COUNT to the number of scalar latch iterations. Peeling + for alignment by masking shifts the active lanes up, and we cannot tell + yet whether peeling will be done by masking, or whether masks will be + needed at all, so be conservative and do not lower COUNT when peeling + for alignment. */ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); widest_int max_iters; - if (max_loop_iterations (loop, &max_iters) + if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) + && max_loop_iterations (loop, &max_iters) && max_iters < count) count = max_iters.to_shwi ();
