Without partial vectors, the vector IV step (VF) and the skip-vector bound
(VF - 1, plus 1 when peeling for gaps) are built as constants of the NITERS
type. With an 8-bit loop counter and a VF of [16, 16], a 2048-bit SVE
vector makes the VF 256 == 0, so the vector loop is never skipped, the IV
steps by 0 and the loop runs off the end of its arrays. niters_vf_2.c
fails this way at plain -O3; GCC 16 is affected too. When partial vectors
are not used and the maximum VF (bounded by
targetm.poly_int_indeterminate_bound when the target has it) may not fit
the NITERS type, convert NITERSM1 to sizetype and recompute NITERS from it,
so that a count of 2^N does not wrap either. Masked loops keep the narrow
type; widening only adds zero-extensions there.
Bootstrapped and regression tested on x86_64-pc-linux-gnu; regression tested
on aarch64-linux-gnu (qemu, VL 128 and SVE VL 2048).
Assisted-by: Claude Opus 5 (Anthropic)
gcc/ChangeLog:
* tree-vect-loop.cc (vect_determine_partial_vectors_and_peeling):
Widen NITERSM1 and NITERS to sizetype if the maximum VF may not fit
their type and partial vectors are not used.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/niters_vf_1.c: New test.
* gcc.target/aarch64/sve/niters_vf_1_run.c: New test.
* gcc.target/aarch64/sve/niters_vf_2.c: New test.
Signed-off-by: Dominic P <[email protected]>
---
Changes in v2: widen the NITERS type instead of failing, as Richard S.
suggested. I used sizetype: on the aarch64 SVE tests (-O3, partial
vectors off) it gives 7728 instructions against 7807 for unsigned int
(pristine 7960). Masked loops keep the narrow type, since widening only
adds zero-extensions there (struct_vect_7.c).
Richard B.: the .WHILE_ULT compare type only covers masked loops. The
failing loops use full vectors, where the IV and the skip guard are built
in the NITERS type; in niters_vf_2.c the main loop has "unsigned char
ivtmp = ivtmp + POLY_INT_CST [32, 32]" and only its masked epilogue uses
.WHILE_ULT.
The loops v1 rejected (and RISC-V poly_licm-3.c) stay vectorised.
Advanced SIMD and x86_64 output is unchanged on the vect and target
testsuites.
OK for trunk?
.../gcc.target/aarch64/sve/niters_vf_1.c | 26 ++++++++++++++++
.../gcc.target/aarch64/sve/niters_vf_1_run.c | 26 ++++++++++++++++
.../gcc.target/aarch64/sve/niters_vf_2.c | 31 +++++++++++++++++++
gcc/tree-vect-loop.cc | 25 +++++++++++++++
4 files changed, 108 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1_run.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/niters_vf_2.c
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1.c
new file mode 100644
index 00000000000..59872fe0cb7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -msve-vector-bits=scalable -fno-vect-cost-model --param
vect-partial-vector-usage=0 -fdump-tree-vect-details" } */
+
+#include <stdint.h>
+
+uint8_t in[1024];
+uint8_t out[256];
+
+uint32_t __attribute__ ((noipa))
+f1 (int n)
+{
+ uint32_t r = 7;
+ for (uint8_t i = 0; i < (uint8_t) n; i++)
+ r += in[i * 4 + 1];
+ return r;
+}
+
+void __attribute__ ((noipa))
+f2 (int n)
+{
+ for (uint8_t i = 0; i < (uint8_t) n; i++)
+ out[i] = in[i * 2] + 1;
+}
+
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" } } */
+/* { dg-final { scan-tree-dump-not {unsigned char niters\.} "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1_run.c
new file mode 100644
index 00000000000..eee9bcfb4dd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_1_run.c
@@ -0,0 +1,26 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O3 -msve-vector-bits=scalable -fno-vect-cost-model --param
vect-partial-vector-usage=0" } */
+
+#include "niters_vf_1.c"
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ for (int i = 0; i < 1024; i++)
+ in[i] = i * 7;
+ for (int n = 0; n < 256; n++)
+ {
+ uint32_t r = 7;
+ for (int i = 0; i < n; i++)
+ r += in[i * 4 + 1];
+ if (f1 (n) != r)
+ __builtin_abort ();
+ for (int i = 0; i < 256; i++)
+ out[i] = 0;
+ f2 (n);
+ for (int i = 0; i < 256; i++)
+ if (out[i] != (i < n ? (uint8_t) (in[i * 2] + 1) : 0))
+ __builtin_abort ();
+ }
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_2.c
new file mode 100644
index 00000000000..f5c6dddc726
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/niters_vf_2.c
@@ -0,0 +1,31 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O3" } */
+
+#include <stdint.h>
+
+uint8_t c[512];
+
+void __attribute__ ((noipa))
+f (uint8_t n)
+{
+ uint8_t i = 0;
+#pragma GCC unroll 32
+ do
+ c[i] += 1;
+ while (i++ != n);
+}
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ for (int n = 0; n < 256; n++)
+ {
+ for (int i = 0; i < 512; i++)
+ c[i] = 0;
+ f (n);
+ for (int i = 0; i < 512; i++)
+ if (c[i] != (i <= n))
+ __builtin_abort ();
+ }
+ return 0;
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 9b2afad6034..6e861dc857f 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -2126,6 +2126,31 @@ vect_determine_partial_vectors_and_peeling
(loop_vec_info loop_vinfo,
"not vectorized: loop needs but cannot "
"use partial vectors\n");
+ /* Without partial vectors the IV step and the skip-vector bound are
+ built in the type of NITERS, so widen it if it may not hold the VF. */
+ if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
+ && !LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo))
+ {
+ widest_int max_vf = vect_max_vf (loop_vinfo);
+ if (targetm.poly_int_indeterminate_bound)
+ {
+ poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ poly_uint64 bound = targetm.poly_int_indeterminate_bound ();
+ max_vf = vf.coeffs[0];
+ for (unsigned int i = 1; i < NUM_POLY_INT_COEFFS; ++i)
+ max_vf += widest_int (vf.coeffs[i]) * bound.coeffs[i];
+ }
+ tree nitersm1 = LOOP_VINFO_NITERSM1 (loop_vinfo);
+ if (wi::min_precision (max_vf, UNSIGNED)
+ > TYPE_PRECISION (TREE_TYPE (nitersm1)))
+ {
+ nitersm1 = fold_convert (sizetype, nitersm1);
+ LOOP_VINFO_NITERSM1 (loop_vinfo) = nitersm1;
+ LOOP_VINFO_NITERS (loop_vinfo)
+ = size_binop (PLUS_EXPR, nitersm1, size_one_node);
+ }
+ }
+
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"operating on %s vectors%s.\n",
--
2.56.0
