gcc/ChangeLog:
* tree-vect-loop.cc (_loop_vec_info::_loop_vec_info): Initialize
can_ffr to true.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/ffr_1.c: New test.
* gcc.target/aarch64/sve/ffr_2.c: New test.
* gcc.target/aarch64/sve/ffr_3.c: New test.
* gcc.target/aarch64/sve/ffr_4.c: New test.
* gcc.target/aarch64/sve/ffr_5.c: New test.
* gcc.target/aarch64/sve/ffr_6.c: New test.
* gcc.target/aarch64/sve/ffr_6_run.c: New test.
* gcc.target/aarch64/sve/ffr_7.c: New test.
* gcc.target/aarch64/sve/ffr_8.c: New test.
* gcc.target/aarch64/sve/ffr_9.c: New test.
---
gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c | 16 ++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c | 35 ++++++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c | 42 ++++++++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c | 25 ++++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c | 25 ++++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c | 43 ++++++++++
.../gcc.target/aarch64/sve/ffr_6_run.c | 81 +++++++++++++++++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c | 16 ++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c | 16 ++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c | 18 +++++
gcc/tree-vect-loop.cc | 4 +-
11 files changed, 319 insertions(+), 2 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
new file mode 100644
index 00000000000..d3f4466b86b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps" } */
+
+#include <stdint.h>
+
+int foo (int *restrict a, int *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] == b[i])
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "Will use ffr" 2 "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1w\tz[0-9]+\.s, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
new file mode 100644
index 00000000000..168df43f954
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps" } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+void foo(
+ type * const restrict dst1,
+ type * const restrict dst2,
+ type * const restrict src1,
+ type * const restrict src1b,
+ type * const restrict src2,
+ type * const restrict src3,
+ unsigned int n)
+{
+
+ for (int i = 0; i < n; i++) {
+ type v1 = src1[i];
+ type v1b = src1b[i];
+ if (v1 == v1b) {break;}
+ type v2 = src2[i];
+ dst1[i] = v1 + v2;
+ if (v2 == 1) {break;}
+ type v3 = src3[i];
+ dst2[i] = v1 + v2 + v3;
+ }
+}
+
+/* { dg-final { scan-tree-dump-times "Will use ffr" 3 "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1h\tz[0-9]+\.h, p[0-9]+/z,} 3 } }
*/
+
+// We need two ffr regions here so there should be 2 reads
+
+/* { dg-final { scan-assembler-times {\trdffrs?\tp[0-9]+\.b} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
new file mode 100644
index 00000000000..9cd3b2fdfde
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+type call [[gnu::simd, gnu::const]] (type);
+
+void foo(
+ type * const restrict dst1,
+ type * const restrict src1,
+ type * const restrict src1b,
+ type * const restrict src2,
+ unsigned int n)
+{
+
+ for (int i = 0; i < n; i++) {
+ type v1b = call (src1b[i]);
+ type v1 = src1[i];
+ if (v1 == v1b) {break;}
+ type v2 = src2[i];
+ dst1[i] = v1 + v2;
+ }
+}
+
+/* { dg-final { scan-tree-dump-times "Will use ffr" 1 "vect" } } */
+
+// Make sure the call does not come between the loads and the rdffr
+// As the call can clobber the FFR state
+/*
+** foo:
+** ...
+** ldff1h z[0-9]+\.h, p[0-9]/z, \[x[0-9]+\]
+** ldff1h z[0-9]+\.h, p[0-9]/z, \[x[0-9]+\]
+** ...
+** rdffrs? p[0-9]+.b
+** ...
+** bl _ZGVsMxv_call
+** ...
+*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
new file mode 100644
index 00000000000..9cff4becd13
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps" } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+ #include <stdint.h>
+void
+foo (uint16_t *const restrict dst1,
+ uint8_t *const restrict src1,
+ uint16_t *const restrict src2, unsigned int n)
+{
+ for (int i = 0; i < n && src1[2 * i] + src1[2 * i + 1] != 5; i++)
+ {
+ uint8_t v1 = src1[2 * i];
+ uint8_t v1a = src1[2 * i + 1];
+ uint16_t v2 = src2[i];
+ dst1[i] = v1 + v1a + v2;
+ }
+}
+
+/* This should not be vectorized with FFR as we can't handle multiple lanes.
*/
+/* { dg-final { scan-tree-dump-times "Will use ffr" 0 "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1} 0 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
new file mode 100644
index 00000000000..0165d508f7e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps" } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+void
+foo (uint16_t *const restrict dst1,
+ uint16_t *const restrict src1,
+ uint16_t *const restrict src2,
+ unsigned int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ uint16_t v1 = src1[i];
+ uint16_t v2 = src1[i] == 0 ? 0 : src2[i];
+ if (v1 + v2 == 100) break;
+ dst1[i] = v1 + v2;
+ }
+}
+
+/* This should not be vectorized with FFR as can not mask FFR reads. */
+/* { dg-final { scan-tree-dump-times "Will use ffr" 0 "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1} 0 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
new file mode 100644
index 00000000000..afaa67b886c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
+
+#include <stdint.h>
+
+__attribute__ ((noipa))
+int
+foo_vect (uint16_t *const restrict out,
+ uint16_t *const restrict src1,
+ uint16_t *const restrict src2,
+ unsigned int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ uint16_t v1 = src1[i];
+ uint16_t v2 = src2[i];
+ *out += v1 == v2;
+ if (v1 == 0)
+ return 1;
+ }
+ return 0;
+}
+
+__attribute__ ((noipa))
+int
+foo_no_vect (uint16_t *const restrict out,
+ uint16_t *const restrict src1,
+ uint16_t *const restrict src2,
+ unsigned int n)
+{
+#pragma GCC novector
+ for (int i = 0; i < n; i++)
+ {
+ uint16_t v1 = src1[i];
+ uint16_t v2 = src2[i];
+ *out += v1 == v2;
+ if (v1 == 0)
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-times {\tldff1} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
new file mode 100644
index 00000000000..8c45821946a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
@@ -0,0 +1,81 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+
+#include "ffr_6.c"
+#include <stdint.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#define N 10000
+int main () {
+ uint16_t *b1 = malloc (sizeof(uint16_t) * N);
+ uint16_t *b2 = malloc (sizeof(uint16_t) * N);
+
+ for (int i = 0; i < N; i++) {
+ b1[i] = 1 + (i % 3) * 3;
+ b2[i] = 1 + (i % 11) * 11;
+ /* These should only intersect when i % 7 = 0 and i % 11 = 0 */
+ }
+
+ uint16_t vect_out;
+ uint16_t no_vect_out;
+ int res_vect;
+ int res_no_vect;
+
+ /* Check while aligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1, b2, N);
+ res_no_vect = foo_no_vect(&no_vect_out, b1, b2, N);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while mutually misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+5, b2+5, N-5);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+5, b2+5, N-5);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while independently misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+9, b2+25, N-25);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+9, b2+25, N-25);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* insert an early break and check it still works */
+
+ b1[N/2+75] = 0;
+
+ /* Check while aligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1, b2, N);
+ res_no_vect = foo_no_vect(&no_vect_out, b1, b2, N);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while mutually misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+5, b2+5, N-5);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+5, b2+5, N-5);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while independently misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+9, b2+25, N-25);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+9, b2+25, N-25);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
new file mode 100644
index 00000000000..02b0481b41b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
-mtune=generic-armv9-a --param=vect-ffr-usage=2" } */
+// Options to try get the cost model to select VNx4SI
+
+#include <stdint.h>
+
+int foo (char *restrict a, char *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] + b[i] == 0)
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-times {\tldff1b\tz[0-9]+\.s, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
new file mode 100644
index 00000000000..5947dd92349
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps -mtune=generic
--param=vect-ffr-usage=2" } */
+// Options to get the cost model to select VNx8HI
+
+#include <stdint.h>
+
+int foo (char *restrict a, char *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] + b[i] == 0)
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-times {\tldff1b\tz[0-9]+\.h, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
new file mode 100644
index 00000000000..45ccaa2fac2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-vect-details --save-temps" } */
+
+/* Check we use FFR at O2. */
+
+#include <stdint.h>
+
+int foo (int *restrict a, int *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] == b[i])
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "Will use ffr" 2 "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1w\tz[0-9]+\.s, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index afebd7e5fa7..9afac8841df 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -757,7 +757,7 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in,
vec_info_shared *shared)
can_use_partial_vectors_p (true),
must_use_partial_vectors_p (false),
must_use_ffr_p (false),
- can_use_ffr_p (false),
+ can_use_ffr_p (true),
using_ffr_p (false),
using_partial_vectors_p (false),
using_decrementing_iv_p (false),
@@ -2912,7 +2912,7 @@ again:
LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
LOOP_VINFO_USING_FFR_P (loop_vinfo) = false;
LOOP_VINFO_MUST_USE_FFR_P (loop_vinfo) = false;
- LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = false;
+ LOOP_VINFO_CAN_USE_FFR_P (loop_vinfo) = true;
LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) = false;
LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo) = false;
LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo) = false;
--
2.34.1
