Consider the following simple loop:
void foo (int *x)
{
for (int i = 0; i < 1000; i++)
x[i] *= 2;
}
compiled at -Ofast -march=armv8-a+sve generates
foo:
mov w1, 0
cntw x3
mov w2, 1000
whilelo p7.s, wzr, w2
.L2:
ld1w z31.s, p7/z, [x0, x1, lsl 2]
add z31.s, z31.s, z31.s
st1w z31.s, p7, [x0, x1, lsl 2]
add x1, x1, x3
whilelo p7.s, w1, w2
b.any .L2
ret
Which is nice, but the whilelo in the pre-header is unneeded. Due to the
architecturally defined minimum and maximum vector lengths[1] we know that at
any vector length the predicate is an all lanes active predicate, i.e. p7 is
always ptrue.
We can use gimple-isel these days while we still have range information on
the operands of .WHILE_ULTs to do this folding.
As such this patch folds whenever possible WHILE_ULTs into ptrue which are
cheaper to execute and so lowers our costs for entering the loops. i.e. the
above generates:
foo:
mov w1, 0
cntw x3
mov w2, 1000
ptrue p7.b, all
.L2:
ld1w z31.s, p7/z, [x0, x1, lsl 2]
add z31.s, z31.s, z31.s
st1w z31.s, p7, [x0, x1, lsl 2]
add x1, x1, x3
whilelo p7.s, w1, w2
b.any .L2
ret
[1] https://developer.arm.com/documentation/102476/0101/Introducing-SVE
Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
Any feedback? otherwise will commit early next week.
Thanks,
Tamar
gcc/ChangeLog:
* config/aarch64/aarch64.cc (aarch64_fold_while_ult_to_ptrue): New.
(aarch64_instruction_selection): Use it.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/fmaxnm_2.c: Update output.
* gcc.target/aarch64/sve/fmaxnm_3.c: Likewise.
* gcc.target/aarch64/sve/fminnm_2.c: Likewise.
* gcc.target/aarch64/sve/fminnm_3.c: Likewise.
* gcc.target/aarch64/sve/slp_12.c: Likewise.
* gcc.target/aarch64/sve/unpacked_fadd_2.c: Likewise.
* gcc.target/aarch64/sve/unpacked_fmul_2.c: Likewise.
* gcc.target/aarch64/sve/unpacked_fsubr_2.c: Likewise.
* gfortran.dg/pr88833.f90: Likewise.
* gcc.target/aarch64/sve/while_ult_1.c: New test.
* gcc.target/aarch64/sve/while_ult_1_run.c: New test.
---
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index
7a81be8dcbe942ca113f7db2fbab4c357c40f462..630856f95281b4e2c290353b3b491404b5dfe633
100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -4442,15 +4442,87 @@ aarch64_fold_sve_ptrue_vl (tree vectype, unsigned int
vl,
return builder.build ();
}
+/* Fold a WHILE_ULT whose bounds are known enough to be expressed as a PTRUE.
+ Generic folding cannot do this because it cannot assume an architectural
+ maximum for a scalable vector. */
+
+static bool
+aarch64_fold_while_ult_to_ptrue (function *fun, gcall *call,
+ gimple_stmt_iterator *gsi)
+{
+ if (!gimple_call_internal_p (call, IFN_WHILE_ULT))
+ return false;
+
+ tree lhs = gimple_call_lhs (call);
+ if (!lhs)
+ return false;
+
+ tree lhs_type = TREE_TYPE (lhs);
+ if (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
+ || !aarch64_sve_pred_mode_p (TYPE_MODE (lhs_type)))
+ return false;
+
+ int_range_max min, max;
+ range_query *query = get_range_query (fun);
+ if (!query->range_of_expr (min, gimple_call_arg (call, 0), call)
+ || !query->range_of_expr (max, gimple_call_arg (call, 1), call)
+ || min.undefined_p ()
+ || max.undefined_p ()
+ || !min.nonnegative_p ()
+ || !max.nonnegative_p ())
+ return false;
+
+ widest_int min_upper = widest_int::from (min.upper_bound (), UNSIGNED);
+ widest_int min_lower = widest_int::from (min.lower_bound (), UNSIGNED);
+ widest_int max_lower = widest_int::from (max.lower_bound (), UNSIGNED);
+ widest_int max_upper = widest_int::from (max.upper_bound (), UNSIGNED);
+ if (wi::leu_p (max_lower, min_upper))
+ return false;
+
+ unsigned int min_nelts
+ = constant_lower_bound (TYPE_VECTOR_SUBPARTS (lhs_type));
+ widest_int max_sve_nelts = min_nelts * 16;
+ widest_int min_gap = max_lower - min_upper;
+ widest_int max_gap = max_upper - min_lower;
+
+ tree pred_cst = NULL_TREE;
+ unsigned HOST_WIDE_INT gap = 0;
+ if (min_gap == max_gap
+ && wi::fits_uhwi_p (min_gap))
+ {
+ gap = min_gap.to_uhwi ();
+ machine_mode pred_mode = TYPE_MODE (lhs_type);
+ if (gap <= UINT_MAX
+ && (aarch64_svpattern_for_vl (pred_mode, (int) gap)
+ != AARCH64_NUM_SVPATTERNS))
+ pred_cst = aarch64_fold_sve_ptrue_vl (lhs_type,
+ (unsigned int) gap, 1);
+ }
+
+ if (!pred_cst)
+ {
+ if (wi::ltu_p (min_gap, max_sve_nelts))
+ return false;
+ pred_cst = build_all_ones_cst (lhs_type);
+ }
+
+ gassign *assign = gimple_build_assign (lhs, pred_cst);
+ gsi_replace (gsi, assign, false);
+ return true;
+}
+
/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
change generic sequences to a form AArch64 has an easier time expanding
instructions for. It's not supposed to be used for generic rewriting that
all targets would benefit from. */
static bool
-aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
+aarch64_instruction_selection (function *fun, gimple_stmt_iterator *gsi)
{
auto stmt = gsi_stmt (*gsi);
+ if (gcall *call = dyn_cast<gcall *> (stmt))
+ return aarch64_fold_while_ult_to_ptrue (fun, call, gsi);
+
gassign *assign = dyn_cast<gassign *> (stmt);
if (!assign)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
index
ee3cdc20f96550ceeb1e6c11dd464c59eb4b3e6f..3f1dfdd60d238945e2c4fbf4676d3e943cfb3e30
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
@@ -16,7 +16,7 @@ f2 (double x, double *ptr)
return x;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s,
\1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler-not
{\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n}
} } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7],
z[0-9]+\.s\n} 1 } } */
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d,
\1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not
{\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n}
} } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7],
z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
index
a8eee0f4b2698f0425f55ea91c523e462ca134bd..2548e55f2841d370d5d30bce23350d3750f094a9
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
@@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
res[1] = x1;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d,
\1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not
{\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n}
} } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7],
z[0-9]+\.d\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
index
10aced05f1afe69f6e646ac23bac8196c13f8357..ed657d58fdfe663f9ddf5b013aa73e14868dbe03
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
@@ -16,7 +16,7 @@ f2 (double x, double *ptr)
return x;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s,
\1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler-not
{\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n}
} } */
/* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7],
z[0-9]+\.s\n} 1 } } */
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d,
\1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not
{\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n}
} } */
/* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7],
z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
index
80ad01602491d07b338113f34d372b5f3970a3be..1f2fc76640fb6c5389c0185a793ca94446ff0352
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
@@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
res[1] = x1;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d,
\1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not
{\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n}
} } */
/* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7],
z[0-9]+\.d\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
index
0b9f8d9bdfaaf868ba0bd73ae83894c68e1613f8..6ba4c9651b67d7e2495b58ffc0ed01f563c428b6
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
@@ -47,9 +47,9 @@ TEST_ALL (VEC_PERM)
/* We should use WHILEs for all accesses. */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b} 20 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 20 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 30 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 30 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 18 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 27 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 24 } } */
/* 6 for the 8-bit types and 2 for the 16-bit types. */
/* { dg-final { scan-assembler-times {\tuqdecb\t} 8 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
index
7a74efd129cb1c2bab6f60b26f371d9a0deb3b1d..7ebee0fff77602c0d63a43f00c4be958c1fe5036
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 15 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
index
eb05600e98a052fcd21f9c7f0f29502bbb5484a5..98617a90260a5d0508fe7c59a76a4359c9e18283
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
index
de9325ccacb00a970ecddd90fb25d609467f74c3..286a1282c402b16bc0733b9c5ca9dbe1c1fb8538
100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
new file mode 100644
index
0000000000000000000000000000000000000000..ddc28029117dadac0610da3ba2753bc9cc4cc385
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
@@ -0,0 +1,99 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve
-msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only
-fdump-tree-vect-details --save-temps" } */
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 14 "vect" } } */
+
+#include <arm_sve.h>
+#include <stdint.h>
+
+#define NOIPA __attribute__ ((noipa))
+
+#define ADD_LOOP(TYPE, NAME, N) \
+ void NOIPA \
+ NAME (TYPE *__restrict a, const TYPE *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < N; ++i) \
+ a[i] = b[i] + (TYPE) 1; \
+ }
+
+ADD_LOOP (uint8_t, full_b, 300)
+ADD_LOOP (uint16_t, full_h, 1000)
+ADD_LOOP (uint32_t, full_s, 100)
+ADD_LOOP (uint64_t, full_d, 40)
+
+ADD_LOOP (uint8_t, exact_b, 256)
+ADD_LOOP (uint16_t, exact_h, 128)
+ADD_LOOP (uint32_t, exact_s, 64)
+ADD_LOOP (uint64_t, exact_d, 32)
+
+svbool_t NOIPA
+finite_b (void)
+{
+ return svptrue_pat_b8 (SV_VL8);
+}
+
+svbool_t NOIPA
+finite_h (void)
+{
+ return svptrue_pat_b16 (SV_VL4);
+}
+
+svbool_t NOIPA
+finite_s (void)
+{
+ return svptrue_pat_b32 (SV_VL2);
+}
+
+svbool_t NOIPA
+finite_d (void)
+{
+ return svptrue_pat_b64 (SV_VL1);
+}
+
+
+ADD_LOOP (uint8_t, partial_b, 255)
+ADD_LOOP (uint16_t, partial_h, 127)
+ADD_LOOP (uint32_t, partial_s, 63)
+ADD_LOOP (uint64_t, partial_d, 31)
+
+void NOIPA
+full_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
+ unsigned int n)
+{
+ if (n < 1000)
+ __builtin_unreachable ();
+ for (unsigned int i = 0; i < n; ++i)
+ a[i] = b[i] + (uint16_t) 1;
+}
+
+void NOIPA
+partial_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
+ unsigned int n)
+{
+ if (n < 100 || n > 127)
+ __builtin_unreachable ();
+ for (unsigned int i = 0; i < n; ++i)
+ a[i] = b[i] + (uint16_t) 1;
+}
+
+/* The first WHILELO in the "full" loops should fold to an all-true
+ predicate. GCC prints all-true predicates as .b, regardless of the
+ element size of the consuming instruction. */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, all\n} 9 } } */
+
+/* The finite loops should fold to finite PTRUE patterns. */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl8\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl4\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl2\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl1\n} 1 } } */
+
+/* The "partial" loops should keep their initial WHILELO. */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, wzr, w[0-9]+} 1 }
} */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, wzr, w[0-9]+} 2 }
} */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, wzr, w[0-9]+} 1 }
} */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, wzr, w[0-9]+} 1 }
} */
+
+/* All loops still need a latch WHILELO. */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, [wx][0-9]+,
[wx][0-9]+} 3 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, [wx][0-9]+,
[wx][0-9]+} 5 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, [wx][0-9]+,
[wx][0-9]+} 3 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, [wx][0-9]+,
[wx][0-9]+} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
new file mode 100644
index
0000000000000000000000000000000000000000..342b11afce92f567c90ab822f7bdc256c51de7cb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
@@ -0,0 +1,66 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve
-msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only" } */
+
+#include "while_ult_1.c"
+
+#define CHECK_LOOP(TYPE, NAME, N) \
+ do \
+ { \
+ TYPE a[N], b[N]; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = 0; \
+ b[i] = (TYPE) (i * 3 + i % 7); \
+ } \
+ NAME (a, b); \
+ for (unsigned int i = 0; i < N; ++i) \
+ if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
+ __builtin_abort (); \
+ } \
+ while (0)
+
+#define CHECK_LOOP_ARG(TYPE, NAME, N) \
+ do \
+ { \
+ TYPE a[N], b[N]; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = 0; \
+ b[i] = (TYPE) (i * 3 + i % 7); \
+ } \
+ NAME (a, b, N); \
+ for (unsigned int i = 0; i < N; ++i) \
+ if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
+ __builtin_abort (); \
+ } \
+ while (0)
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ CHECK_LOOP (uint8_t, full_b, 300);
+ CHECK_LOOP (uint16_t, full_h, 1000);
+ CHECK_LOOP (uint32_t, full_s, 100);
+ CHECK_LOOP (uint64_t, full_d, 40);
+
+ CHECK_LOOP (uint8_t, exact_b, 256);
+ CHECK_LOOP (uint16_t, exact_h, 128);
+ CHECK_LOOP (uint32_t, exact_s, 64);
+ CHECK_LOOP (uint64_t, exact_d, 32);
+
+ svbool_t pg = svptrue_b8 ();
+ if (svcntp_b8 (pg, finite_b ()) != 8
+ || svcntp_b16 (pg, finite_h ()) != 4
+ || svcntp_b32 (pg, finite_s ()) != 2
+ || svcntp_b64 (pg, finite_d ()) != 1)
+ __builtin_abort ();
+
+ CHECK_LOOP (uint8_t, partial_b, 255);
+ CHECK_LOOP (uint16_t, partial_h, 127);
+ CHECK_LOOP (uint32_t, partial_s, 63);
+ CHECK_LOOP (uint64_t, partial_d, 31);
+
+ CHECK_LOOP_ARG (uint16_t, full_range_h, 1000);
+ CHECK_LOOP_ARG (uint16_t, partial_range_h, 127);
+ return 0;
+}
diff --git a/gcc/testsuite/gfortran.dg/pr88833.f90
b/gcc/testsuite/gfortran.dg/pr88833.f90
index
224e6ce5f3d53135c588b6ac0ab6ee1733eab852..5959cb1d16bf3da90bbbdedcb57631610cfc17d0
100644
--- a/gcc/testsuite/gfortran.dg/pr88833.f90
+++ b/gcc/testsuite/gfortran.dg/pr88833.f90
@@ -6,4 +6,4 @@ subroutine foo(x)
x = x + 10
end subroutine foo
-! { dg-final { scan-assembler {\twhilelo\tp[0-9]+\.s, wzr,
(w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
+! { dg-final { scan-assembler-not {\twhilelo\tp[0-9]+\.s, wzr,
(w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
--
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 7a81be8dcbe942ca113f7db2fbab4c357c40f462..630856f95281b4e2c290353b3b491404b5dfe633 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -4442,15 +4442,87 @@ aarch64_fold_sve_ptrue_vl (tree vectype, unsigned int vl,
return builder.build ();
}
+/* Fold a WHILE_ULT whose bounds are known enough to be expressed as a PTRUE.
+ Generic folding cannot do this because it cannot assume an architectural
+ maximum for a scalable vector. */
+
+static bool
+aarch64_fold_while_ult_to_ptrue (function *fun, gcall *call,
+ gimple_stmt_iterator *gsi)
+{
+ if (!gimple_call_internal_p (call, IFN_WHILE_ULT))
+ return false;
+
+ tree lhs = gimple_call_lhs (call);
+ if (!lhs)
+ return false;
+
+ tree lhs_type = TREE_TYPE (lhs);
+ if (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
+ || !aarch64_sve_pred_mode_p (TYPE_MODE (lhs_type)))
+ return false;
+
+ int_range_max min, max;
+ range_query *query = get_range_query (fun);
+ if (!query->range_of_expr (min, gimple_call_arg (call, 0), call)
+ || !query->range_of_expr (max, gimple_call_arg (call, 1), call)
+ || min.undefined_p ()
+ || max.undefined_p ()
+ || !min.nonnegative_p ()
+ || !max.nonnegative_p ())
+ return false;
+
+ widest_int min_upper = widest_int::from (min.upper_bound (), UNSIGNED);
+ widest_int min_lower = widest_int::from (min.lower_bound (), UNSIGNED);
+ widest_int max_lower = widest_int::from (max.lower_bound (), UNSIGNED);
+ widest_int max_upper = widest_int::from (max.upper_bound (), UNSIGNED);
+ if (wi::leu_p (max_lower, min_upper))
+ return false;
+
+ unsigned int min_nelts
+ = constant_lower_bound (TYPE_VECTOR_SUBPARTS (lhs_type));
+ widest_int max_sve_nelts = min_nelts * 16;
+ widest_int min_gap = max_lower - min_upper;
+ widest_int max_gap = max_upper - min_lower;
+
+ tree pred_cst = NULL_TREE;
+ unsigned HOST_WIDE_INT gap = 0;
+ if (min_gap == max_gap
+ && wi::fits_uhwi_p (min_gap))
+ {
+ gap = min_gap.to_uhwi ();
+ machine_mode pred_mode = TYPE_MODE (lhs_type);
+ if (gap <= UINT_MAX
+ && (aarch64_svpattern_for_vl (pred_mode, (int) gap)
+ != AARCH64_NUM_SVPATTERNS))
+ pred_cst = aarch64_fold_sve_ptrue_vl (lhs_type,
+ (unsigned int) gap, 1);
+ }
+
+ if (!pred_cst)
+ {
+ if (wi::ltu_p (min_gap, max_sve_nelts))
+ return false;
+ pred_cst = build_all_ones_cst (lhs_type);
+ }
+
+ gassign *assign = gimple_build_assign (lhs, pred_cst);
+ gsi_replace (gsi, assign, false);
+ return true;
+}
+
/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
change generic sequences to a form AArch64 has an easier time expanding
instructions for. It's not supposed to be used for generic rewriting that
all targets would benefit from. */
static bool
-aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
+aarch64_instruction_selection (function *fun, gimple_stmt_iterator *gsi)
{
auto stmt = gsi_stmt (*gsi);
+ if (gcall *call = dyn_cast<gcall *> (stmt))
+ return aarch64_fold_while_ult_to_ptrue (fun, call, gsi);
+
gassign *assign = dyn_cast<gassign *> (stmt);
if (!assign)
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
index ee3cdc20f96550ceeb1e6c11dd464c59eb4b3e6f..3f1dfdd60d238945e2c4fbf4676d3e943cfb3e30 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
@@ -16,7 +16,7 @@ f2 (double x, double *ptr)
return x;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
index a8eee0f4b2698f0425f55ea91c523e462ca134bd..2548e55f2841d370d5d30bce23350d3750f094a9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
@@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
res[1] = x1;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
index 10aced05f1afe69f6e646ac23bac8196c13f8357..ed657d58fdfe663f9ddf5b013aa73e14868dbe03 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
@@ -16,7 +16,7 @@ f2 (double x, double *ptr)
return x;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
/* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
index 80ad01602491d07b338113f34d372b5f3970a3be..1f2fc76640fb6c5389c0185a793ca94446ff0352 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
@@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
res[1] = x1;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
index 0b9f8d9bdfaaf868ba0bd73ae83894c68e1613f8..6ba4c9651b67d7e2495b58ffc0ed01f563c428b6 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
@@ -47,9 +47,9 @@ TEST_ALL (VEC_PERM)
/* We should use WHILEs for all accesses. */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b} 20 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 20 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 30 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 30 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 18 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 27 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 24 } } */
/* 6 for the 8-bit types and 2 for the 16-bit types. */
/* { dg-final { scan-assembler-times {\tuqdecb\t} 8 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
index 7a74efd129cb1c2bab6f60b26f371d9a0deb3b1d..7ebee0fff77602c0d63a43f00c4be958c1fe5036 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 15 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
index eb05600e98a052fcd21f9c7f0f29502bbb5484a5..98617a90260a5d0508fe7c59a76a4359c9e18283 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
index de9325ccacb00a970ecddd90fb25d609467f74c3..286a1282c402b16bc0733b9c5ca9dbe1c1fb8538 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
new file mode 100644
index 0000000000000000000000000000000000000000..ddc28029117dadac0610da3ba2753bc9cc4cc385
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
@@ -0,0 +1,99 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only -fdump-tree-vect-details --save-temps" } */
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 14 "vect" } } */
+
+#include <arm_sve.h>
+#include <stdint.h>
+
+#define NOIPA __attribute__ ((noipa))
+
+#define ADD_LOOP(TYPE, NAME, N) \
+ void NOIPA \
+ NAME (TYPE *__restrict a, const TYPE *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < N; ++i) \
+ a[i] = b[i] + (TYPE) 1; \
+ }
+
+ADD_LOOP (uint8_t, full_b, 300)
+ADD_LOOP (uint16_t, full_h, 1000)
+ADD_LOOP (uint32_t, full_s, 100)
+ADD_LOOP (uint64_t, full_d, 40)
+
+ADD_LOOP (uint8_t, exact_b, 256)
+ADD_LOOP (uint16_t, exact_h, 128)
+ADD_LOOP (uint32_t, exact_s, 64)
+ADD_LOOP (uint64_t, exact_d, 32)
+
+svbool_t NOIPA
+finite_b (void)
+{
+ return svptrue_pat_b8 (SV_VL8);
+}
+
+svbool_t NOIPA
+finite_h (void)
+{
+ return svptrue_pat_b16 (SV_VL4);
+}
+
+svbool_t NOIPA
+finite_s (void)
+{
+ return svptrue_pat_b32 (SV_VL2);
+}
+
+svbool_t NOIPA
+finite_d (void)
+{
+ return svptrue_pat_b64 (SV_VL1);
+}
+
+
+ADD_LOOP (uint8_t, partial_b, 255)
+ADD_LOOP (uint16_t, partial_h, 127)
+ADD_LOOP (uint32_t, partial_s, 63)
+ADD_LOOP (uint64_t, partial_d, 31)
+
+void NOIPA
+full_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
+ unsigned int n)
+{
+ if (n < 1000)
+ __builtin_unreachable ();
+ for (unsigned int i = 0; i < n; ++i)
+ a[i] = b[i] + (uint16_t) 1;
+}
+
+void NOIPA
+partial_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
+ unsigned int n)
+{
+ if (n < 100 || n > 127)
+ __builtin_unreachable ();
+ for (unsigned int i = 0; i < n; ++i)
+ a[i] = b[i] + (uint16_t) 1;
+}
+
+/* The first WHILELO in the "full" loops should fold to an all-true
+ predicate. GCC prints all-true predicates as .b, regardless of the
+ element size of the consuming instruction. */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, all\n} 9 } } */
+
+/* The finite loops should fold to finite PTRUE patterns. */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl8\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl4\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl2\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl1\n} 1 } } */
+
+/* The "partial" loops should keep their initial WHILELO. */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, wzr, w[0-9]+} 1 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, wzr, w[0-9]+} 2 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, wzr, w[0-9]+} 1 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, wzr, w[0-9]+} 1 } } */
+
+/* All loops still need a latch WHILELO. */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, [wx][0-9]+, [wx][0-9]+} 3 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, [wx][0-9]+, [wx][0-9]+} 5 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, [wx][0-9]+, [wx][0-9]+} 3 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, [wx][0-9]+, [wx][0-9]+} 3 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
new file mode 100644
index 0000000000000000000000000000000000000000..342b11afce92f567c90ab822f7bdc256c51de7cb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
@@ -0,0 +1,66 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only" } */
+
+#include "while_ult_1.c"
+
+#define CHECK_LOOP(TYPE, NAME, N) \
+ do \
+ { \
+ TYPE a[N], b[N]; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = 0; \
+ b[i] = (TYPE) (i * 3 + i % 7); \
+ } \
+ NAME (a, b); \
+ for (unsigned int i = 0; i < N; ++i) \
+ if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
+ __builtin_abort (); \
+ } \
+ while (0)
+
+#define CHECK_LOOP_ARG(TYPE, NAME, N) \
+ do \
+ { \
+ TYPE a[N], b[N]; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = 0; \
+ b[i] = (TYPE) (i * 3 + i % 7); \
+ } \
+ NAME (a, b, N); \
+ for (unsigned int i = 0; i < N; ++i) \
+ if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
+ __builtin_abort (); \
+ } \
+ while (0)
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ CHECK_LOOP (uint8_t, full_b, 300);
+ CHECK_LOOP (uint16_t, full_h, 1000);
+ CHECK_LOOP (uint32_t, full_s, 100);
+ CHECK_LOOP (uint64_t, full_d, 40);
+
+ CHECK_LOOP (uint8_t, exact_b, 256);
+ CHECK_LOOP (uint16_t, exact_h, 128);
+ CHECK_LOOP (uint32_t, exact_s, 64);
+ CHECK_LOOP (uint64_t, exact_d, 32);
+
+ svbool_t pg = svptrue_b8 ();
+ if (svcntp_b8 (pg, finite_b ()) != 8
+ || svcntp_b16 (pg, finite_h ()) != 4
+ || svcntp_b32 (pg, finite_s ()) != 2
+ || svcntp_b64 (pg, finite_d ()) != 1)
+ __builtin_abort ();
+
+ CHECK_LOOP (uint8_t, partial_b, 255);
+ CHECK_LOOP (uint16_t, partial_h, 127);
+ CHECK_LOOP (uint32_t, partial_s, 63);
+ CHECK_LOOP (uint64_t, partial_d, 31);
+
+ CHECK_LOOP_ARG (uint16_t, full_range_h, 1000);
+ CHECK_LOOP_ARG (uint16_t, partial_range_h, 127);
+ return 0;
+}
diff --git a/gcc/testsuite/gfortran.dg/pr88833.f90 b/gcc/testsuite/gfortran.dg/pr88833.f90
index 224e6ce5f3d53135c588b6ac0ab6ee1733eab852..5959cb1d16bf3da90bbbdedcb57631610cfc17d0 100644
--- a/gcc/testsuite/gfortran.dg/pr88833.f90
+++ b/gcc/testsuite/gfortran.dg/pr88833.f90
@@ -6,4 +6,4 @@ subroutine foo(x)
x = x + 10
end subroutine foo
-! { dg-final { scan-assembler {\twhilelo\tp[0-9]+\.s, wzr, (w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
+! { dg-final { scan-assembler-not {\twhilelo\tp[0-9]+\.s, wzr, (w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
