Loop masks and lengths for a grouped load cover whole groups, rounded
up to whole vectors. get_load_store_type does not peel for gaps when
the trailing gap is below the known alignment or when the unmasked
access can be shortened, but a masked final iteration still reads past
the last element used: for long *b, s += b[3*i] + b[3*i+1] faults at
-msve-vector-bits=256. With a non-power-of-two VF from #pragma GCC
unroll the rounded vectors do not cover whole iterations and
vect_verify_full_masking ICEs in exact_div (x86-64-v4, znver5, SVE).
Disable partial vectors when such an overrun is not peeled, and when
the vectors do not cover a whole number of iterations.
Bootstrapped and regtested on x86_64-pc-linux-gnu, and regtested on
aarch64-linux-gnu (qemu, also with -msve-vector-bits=256).
Assisted-by: Claude Opus 5 (Anthropic)
gcc/ChangeLog:
* tree-vect-stmts.cc (check_load_store_for_partial_vectors): Do not
use partial vectors if the vectors do not cover a whole number of
iterations.
(get_load_store_type): Do not use partial vectors if an overrun of
the group is not handled by peeling for gaps.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-partial-gap-1.c: New test.
* gcc.dg/vect/vect-partial-gap-2.c: New test.
Signed-off-by: Dominic P <[email protected]>
---
OK for trunk?
GCC 16.1 has the wrong code (vect-partial-gap-2.c faults with it at
-msve-vector-bits=256 and 512); the ICE is new in GCC 17.
.../gcc.dg/vect/vect-partial-gap-1.c | 14 +++
.../gcc.dg/vect/vect-partial-gap-2.c | 106 ++++++++++++++++++
gcc/tree-vect-stmts.cc | 26 ++++-
3 files changed, 144 insertions(+), 2 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-partial-gap-1.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-partial-gap-2.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-partial-gap-1.c
b/gcc/testsuite/gcc.dg/vect/vect-partial-gap-1.c
new file mode 100644
index 00000000000..4a5186a9d4d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-partial-gap-1.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-fvect-cost-model=dynamic" } */
+/* { dg-additional-options "-mavx512vl" { target avx512vl } } */
+/* { dg-additional-options "-march=armv8-a+sve -msve-vector-bits=128" { target
aarch64*-*-* } } */
+
+long long
+f (long long *b, int n)
+{
+ long long s = 0;
+#pragma GCC unroll 3
+ for (int i = 0; i < n; i++)
+ s += b[3 * i] + b[3 * i + 1];
+ return s;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-partial-gap-2.c
b/gcc/testsuite/gcc.dg/vect/vect-partial-gap-2.c
new file mode 100644
index 00000000000..11772d0ceee
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-partial-gap-2.c
@@ -0,0 +1,106 @@
+/* { dg-require-effective-target mmap } */
+/* { dg-additional-options "-march=armv8-a+sve -msve-vector-bits=256" { target
aarch64_sve256_hw } } */
+/* { dg-additional-options "-march=armv8-a+sve -msve-vector-bits=512" { target
aarch64_sve512_hw } } */
+
+#include <sys/mman.h>
+#include <unistd.h>
+#include "tree-vect.h"
+
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+long __attribute__ ((noipa))
+f3 (long *b, int n)
+{
+ long s = 0;
+ for (int i = 0; i < n; i++)
+ s += b[3 * i] + b[3 * i + 1];
+ return s;
+}
+
+long __attribute__ ((noipa))
+f5 (long *b, int n)
+{
+ long s = 0;
+ for (int i = 0; i < n; i++)
+ s += b[5 * i] + b[5 * i + 1] + b[5 * i + 2] + b[5 * i + 3];
+ return s;
+}
+
+int __attribute__ ((noipa))
+f8 (int *b, int n)
+{
+ int s = 0;
+ for (int i = 0; i < n; i++)
+ s += b[8 * i] + b[8 * i + 1] + b[8 * i + 2] + b[8 * i + 3];
+ return s;
+}
+
+long __attribute__ ((noipa))
+f3a (long *b, int n)
+{
+ b = __builtin_assume_aligned (b, 64);
+ long s = 0;
+ for (int i = 0; i < n; i++)
+ s += b[3 * i];
+ return s;
+}
+
+int __attribute__ ((noipa))
+f1 (int *b, int n)
+{
+ static int a[2 * 4096];
+ for (int i = 0; i < n; i++)
+ {
+ a[2 * i] = b[i];
+ a[2 * i + 1] = b[i];
+ }
+ return a[2 * n - 1];
+}
+
+/* Place the last element F reads just below the protected page END,
+ with B aligned to A bytes. */
+#define CHECK(F, T, G, L, A) \
+ for (int n = 1; G * n * sizeof (T) <= 4096; n++) \
+ { \
+ T *b = (T *) end - (G * (n - 1) + L); \
+ T s = 0; \
+ if ((__UINTPTR_TYPE__) b % A) \
+ continue; \
+ _Pragma ("GCC novector") \
+ for (int i = 0; i < G * (n - 1) + L; i++)
\
+ { \
+ b[i] = i; \
+ if (i % G < L) \
+ s += i; \
+ } \
+ if (F (b, n) != s) \
+ __builtin_abort (); \
+ }
+
+int
+main (void)
+{
+ check_vect ();
+
+ long pg = sysconf (_SC_PAGESIZE);
+ char *m = mmap (0, 2 * pg, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (pg < 4096 || m == MAP_FAILED || mprotect (m + pg, pg, PROT_NONE))
+ return 0;
+ char *end = m + pg;
+
+ for (int n = 1; n <= 1024; n++)
+ {
+ int *b = (int *) end - n;
+ b[n - 1] = n;
+ if (f1 (b, n) != n)
+ __builtin_abort ();
+ }
+ CHECK (f3, long, 3, 2, 1)
+ CHECK (f5, long, 5, 4, 1)
+ CHECK (f8, int, 8, 4, 1)
+ CHECK (f3a, long, 3, 1, 64)
+ return 0;
+}
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 244f319d0e4..b4ee0fb2525 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1552,13 +1552,23 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ nvectors = group_memory_nvectors (group_size * vf, nunits);
+ if (!multiple_p (nvectors * nunits, vf))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't operate on partial vectors because the"
+ " vectors do not cover whole iterations.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ return;
+ }
+
machine_mode mask_mode;
machine_mode vmode;
bool using_partial_vectors_p = false;
if (get_len_load_store_mode
(vecmode, is_load, nullptr, elsvals).exists (&vmode))
{
- nvectors = group_memory_nvectors (group_size * vf, nunits);
unsigned factor = (vecmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vecmode);
vect_record_loop_len (loop_vinfo, lens, nvectors, vectype, factor);
using_partial_vectors_p = true;
@@ -1567,7 +1577,6 @@ check_load_store_for_partial_vectors (loop_vec_info
loop_vinfo, tree vectype,
&& can_vec_mask_load_store_p (vecmode, mask_mode, is_load, NULL,
elsvals))
{
- nvectors = group_memory_nvectors (group_size * vf, nunits);
vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype,
scalar_mask);
using_partial_vectors_p = true;
}
@@ -2372,6 +2381,19 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info
stmt_info,
!= NULL_TREE))))
overrun_p = false;
+ if (loop_vinfo
+ && (gap != 0 || large_vector_overrun_p)
+ && !overrun_p
+ && *memory_access_type != VMAT_ELEMENTWISE)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't operate on partial vectors because "
+ "they would read past the end of the "
+ "group\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
+
if (overrun_p && !can_overrun_p)
{
if (dump_enabled_p ())
--
2.55.0
