vect_use_grouped_gather computes the offsets { 0, X, X*2, ... } in the
unsigned element type of the punned vector, so if that is narrower than
a pointer a negative step is zero-extended and a large one wraps.
With -msve-vector-bits=256, a grouped load with a negative run-time
stride faults (ld1w ..., [x4, z31.s, uxtw]). Ask for a signed offset
for a negative constant step, and use the gather only if the returned
offset type is pointer-wide or the constant step's offsets fit it.
The range of the offsets comes from vect_strided_offset_range, split
out of vect_truncate_gather_scatter_offset.
This also fixes vect-alias-check-10.c and -11.c with SVE-256.
Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression
tested on aarch64-linux-gnu (qemu, default and SVE-256 boards).
Assisted-by: Claude Opus 5 (Anthropic)
gcc/ChangeLog:
* tree-vect-stmts.cc (vect_strided_offset_range): New function,
split out of...
(vect_truncate_gather_scatter_offset): ...here. Remove the
overflow note.
(vect_use_grouped_gather): Use a signed offset type for a negative
constant step. Require the offsets not to wrap in the offset type.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-strided-neg-step-1.c: New test.
* gcc.target/aarch64/sve/strided_grouped_1.c: New test.
* gcc.target/aarch64/sve/strided_grouped_1_run.c: New test.
Signed-off-by: Dominic P <[email protected]>
---
.../gcc.dg/vect/vect-strided-neg-step-1.c | 82 ++++++++++++++++++
.../aarch64/sve/strided_grouped_1.c | 8 ++
.../aarch64/sve/strided_grouped_1_run.c | 4 +
gcc/tree-vect-stmts.cc | 85 +++++++++++--------
4 files changed, 143 insertions(+), 36 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-strided-neg-step-1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1_run.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-strided-neg-step-1.c
b/gcc/testsuite/gcc.dg/vect/vect-strided-neg-step-1.c
new file mode 100644
index 00000000000..0d0fe976334
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-strided-neg-step-1.c
@@ -0,0 +1,82 @@
+/* Grouped strided accesses with a negative step. */
+
+#include "tree-vect.h"
+
+typedef unsigned char uc;
+typedef unsigned short us;
+
+#define N 32
+#define S 40
+
+#define TEST(T, G) \
+ void __attribute__ ((noipa)) \
+ load_##T (T *restrict o, T *restrict p, long stride, int n) \
+ { \
+ for (int i = 0; i < n; i++)
\
+ {
\
+ for (int k = 0; k < G; k++) \
+ o[i * G + k] = p[k] * 3; \
+ p += stride; \
+ }
\
+ } \
+ void __attribute__ ((noipa)) \
+ store_##T (T *restrict q, T *restrict a, long stride, int n) \
+ { \
+ for (int i = 0; i < n; i++)
\
+ {
\
+ for (int k = 0; k < G; k++) \
+ q[k] = a[i * G + k] * 3; \
+ q += stride; \
+ }
\
+ } \
+ void __attribute__ ((noipa)) \
+ store_down_##T (T *restrict q, T *restrict a, int n) \
+ { \
+ for (int i = 0; i < n; i++)
\
+ {
\
+ for (int k = 0; k < G; k++) \
+ q[k] = a[i * G + k] * 3; \
+ q -= S; \
+ }
\
+ } \
+ void __attribute__ ((noipa)) \
+ check_##T (void) \
+ { \
+ static T a[N * G], o[N * G], m[N * S]; \
+ T *last = m + (N - 1) * S; \
+ _Pragma ("GCC novector") \
+ for (int i = 0; i < N * S; i++) \
+ m[i] = i * 7 + 1;
\
+ _Pragma ("GCC novector") \
+ for (int i = 0; i < N * G; i++) \
+ a[i] = i * 5 + 2;
\
+ load_##T (o, last, -S, N); \
+ _Pragma ("GCC novector") \
+ for (int i = 0; i < N * G; i++) \
+ if (o[i] != (T) (last[i % G - i / G * S] * 3)) \
+ __builtin_abort (); \
+ store_##T (last, a, -S, N);
\
+ _Pragma ("GCC novector") \
+ for (int i = 0; i < N * G; i++) \
+ if (last[i % G - i / G * S] != (T) (a[i] * 3)) \
+ __builtin_abort (); \
+ for (int i = 0; i < N * G; i++) \
+ a[i] += 1; \
+ store_down_##T (last, a, N); \
+ _Pragma ("GCC novector") \
+ for (int i = 0; i < N * G; i++) \
+ if (last[i % G - i / G * S] != (T) (a[i] * 3)) \
+ __builtin_abort (); \
+ }
+
+TEST (uc, 4)
+TEST (us, 2)
+
+int
+main (void)
+{
+ check_vect ();
+ check_uc ();
+ check_us ();
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1.c
new file mode 100644
index 00000000000..662f9753ecc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1.c
@@ -0,0 +1,8 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O3 -fno-vect-cost-model -msve-vector-bits=256" } */
+
+#include "../../../gcc.dg/vect/vect-strided-neg-step-1.c"
+
+/* { dg-final { scan-assembler-not {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x[0-9]+,
z[0-9]+\.s, uxtw\]} } } */
+/* { dg-final { scan-assembler-not {\tst1w\tz[0-9]+\.s, p[0-7], \[x[0-9]+,
z[0-9]+\.s, uxtw\]} } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x[0-9]+,
z[0-9]+\.s, sxtw\]} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1_run.c
new file mode 100644
index 00000000000..12c245c6ec1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/strided_grouped_1_run.c
@@ -0,0 +1,4 @@
+/* { dg-do run { target { aarch64_sve256_hw && lp64 } } } */
+/* { dg-options "-O3 -fno-vect-cost-model -msve-vector-bits=256" } */
+
+#include "../../../gcc.dg/vect/vect-strided-neg-step-1.c"
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 1db6786078e..7c96e9c8f55 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1613,6 +1613,36 @@ prepare_vec_mask (loop_vec_info loop_vinfo, tree
mask_type, tree loop_mask,
return and_res;
}
+/* Store in *RANGE the last offset of { 0, X, X*2, X*3, ... } for a strided
+ access in LOOP_VINFO, where X is the constant step STEP divided by SCALE.
+ Return false if STEP is not a multiple of SCALE or *RANGE overflows. */
+
+static bool
+vect_strided_offset_range (loop_vec_info loop_vinfo, tree step,
+ unsigned int scale, widest_int *range)
+{
+ /* Set COUNT to the upper limit on the number of elements - 1.
+ Start with the maximum vectorization factor. */
+ unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
+
+ /* Try lowering COUNT to the number of scalar latch iterations.
+ Peeling for alignment by masking shifts the active lanes up. */
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ widest_int max_iters;
+ if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
+ && max_loop_iterations (loop, &max_iters)
+ && max_iters < count)
+ count = max_iters.to_shwi ();
+
+ widest_int factor;
+ if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
+ return false;
+
+ wi::overflow_type overflow;
+ *range = wi::mul (count, factor, SIGNED, &overflow);
+ return !overflow;
+}
+
/* Determine whether we can use a gather load or scatter store to vectorize
strided load or store STMT_INFO by truncating the current offset to a
smaller width. We need to be able to construct an offset vector:
@@ -1645,37 +1675,16 @@ vect_truncate_gather_scatter_offset (stmt_vec_info
stmt_info, tree vectype,
return false;
}
- /* Get the number of bits in an element. */
- scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
- unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
-
- /* Set COUNT to the upper limit on the number of elements - 1.
- Start with the maximum vectorization factor. */
- unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
-
- /* Try lowering COUNT to the number of scalar latch iterations.
- Peeling for alignment by masking shifts the active lanes up. */
- class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- widest_int max_iters;
- if (!LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
- && max_loop_iterations (loop, &max_iters)
- && max_iters < count)
- count = max_iters.to_shwi ();
-
/* Try scales of 1 and the element size. */
unsigned int scales[] = { 1, vect_get_scalar_dr_size (dr_info) };
- wi::overflow_type overflow = wi::OVF_NONE;
for (int i = 0; i < 2; ++i)
{
unsigned int scale = scales[i];
- widest_int factor;
- if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
+ widest_int range;
+ if (!vect_strided_offset_range (loop_vinfo, step, scale, &range))
continue;
- /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
- widest_int range = wi::mul (count, factor, SIGNED, &overflow);
- if (overflow)
- continue;
+ /* Determine the minimum precision of RANGE. */
signop sign = range >= 0 ? UNSIGNED : SIGNED;
unsigned int min_offset_bits = wi::min_precision (range, sign);
@@ -1711,11 +1720,6 @@ vect_truncate_gather_scatter_offset (stmt_vec_info
stmt_info, tree vectype,
return true;
}
- if (overflow && dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "truncating gather/scatter offset to %d bits"
- " might change its value.\n", element_bits);
-
return false;
}
@@ -1762,22 +1766,31 @@ vect_use_grouped_gather (dr_vec_info *dr_info, tree
vectype,
internal_fn ifn;
tree offset_vectype = *pun_vectype;
+ tree step = DR_STEP (dr);
+ if (TREE_CODE (step) == INTEGER_CST && tree_int_cst_sgn (step) < 0)
+ offset_vectype = signed_type_for (offset_vectype);
internal_fn strided_ifn = DR_IS_READ (dr)
? IFN_MASK_LEN_STRIDED_LOAD : IFN_MASK_LEN_STRIDED_STORE;
- /* Check if we have a gather/scatter with the new type. We're just trying
- with the type itself as offset for now. If not, check if we have a
- strided load/store. These have fewer constraints (for example no offset
- type must exist) so it is possible that even though a gather/scatter is
- not available we still have a strided load/store. */
+ /* Check if we have a gather/scatter with the new type as offset whose
+ offsets { 0, X, X*2, ... } for DR_STEP X do not wrap. If not, check
+ if we have a strided load/store. These have fewer constraints (for
+ example no offset type must exist) so it is possible that even though
+ a gather/scatter is not available we still have a strided
+ load/store. */
bool ok = false;
tree tmp_vectype;
int tmp_scale;
+ widest_int range;
if (vect_gather_scatter_fn_p
(loop_vinfo, DR_IS_READ (dr), masked_p, *pun_vectype,
- TREE_TYPE (*pun_vectype), *pun_vectype, 1, &tmp_scale, &ifn,
- &offset_vectype, &tmp_vectype, elsvals))
+ TREE_TYPE (*pun_vectype), offset_vectype, 1, &tmp_scale, &ifn,
+ &offset_vectype, &tmp_vectype, elsvals)
+ && (TYPE_PRECISION (TREE_TYPE (offset_vectype)) >= POINTER_SIZE
+ || (TREE_CODE (step) == INTEGER_CST
+ && vect_strided_offset_range (loop_vinfo, step, 1, &range)
+ && wi::fits_to_tree_p (range, TREE_TYPE (offset_vectype)))))
ok = true;
else if (internal_strided_fn_supported_p (strided_ifn, *pun_vectype,
elsvals))
--
2.55.0
