Add the transformation code required to generate HSSR loops.
This includes adding a "fixup" BB for handling when the HSSR loads return
partial reads.
gcc/ChangeLog:
* tree-vect-loop-manip.cc (vect_get_loop_iv_increment):
Add support for HSSR iv increments.
(vect_add_hssr_read): New function.
(vect_add_hssr_fixup_controls): New function.
* tree-vect-loop.cc (vect_get_loop_mask): Add logic for getting
the governing mask from the HSSR region (if applicable).
(vect_transform_loop): Add logic for HSSR.
* tree-vect-slp.cc (vect_schedule_slp_node): Add logic to make
sure nodes requiring a HSSR mask come after the HSSR loads.
(vect_schedule_slp): Add logic for inserting HSSR read points.
* tree-vect-stmts.cc (vect_finish_replace_stmt): Add logic for
inserting HSSR loads.
(vectorizable_load): Add logic for generating HSSR loads.
* tree-vectorizer.h (LOOP_VINFO_IV_INCREMENT_INVARIANT_P): Add
case for HSSR loops.
(vect_add_hssr_fixup_controls): New function.
(vect_add_hssr_read): New function.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/ffr_1.c: New test.
* gcc.target/aarch64/sve/ffr_11.c: New test.
* gcc.target/aarch64/sve/ffr_12.c: New test.
* gcc.target/aarch64/sve/ffr_13.c: New test.
* gcc.target/aarch64/sve/ffr_14.c: New test.
* gcc.target/aarch64/sve/ffr_2.c: New test.
* gcc.target/aarch64/sve/ffr_3.c: New test.
* gcc.target/aarch64/sve/ffr_4.c: New test.
* gcc.target/aarch64/sve/ffr_5.c: New test.
* gcc.target/aarch64/sve/ffr_6.c: New test.
* gcc.target/aarch64/sve/ffr_6_run.c: New test.
* gcc.target/aarch64/sve/ffr_7.c: New test.
* gcc.target/aarch64/sve/ffr_8.c: New test.
* gcc.target/aarch64/sve/ffr_9.c: New test.
---
gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c | 16 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c | 13 +
gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c | 17 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c | 15 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c | 18 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c | 35 +++
gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c | 44 ++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c | 25 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c | 25 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c | 43 ++++
.../gcc.target/aarch64/sve/ffr_6_run.c | 82 ++++++
gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c | 19 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c | 17 ++
gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c | 18 ++
gcc/tree-vect-loop-manip.cc | 234 ++++++++++++++++++
gcc/tree-vect-loop.cc | 73 +++++-
gcc/tree-vect-slp.cc | 17 ++
gcc/tree-vect-stmts.cc | 61 +++--
gcc/tree-vectorizer.h | 4 +-
19 files changed, 758 insertions(+), 18 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
new file mode 100644
index 00000000000..3027c6d69f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_1.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+
+#include <stdint.h>
+
+int foo (int *restrict a, int *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] == b[i])
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "Will use hssr" "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1w\tz[0-9]+\.s, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
new file mode 100644
index 00000000000..dedea70dd3b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_11.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 --param=vect-hssr-usage=2" } */
+
+// Check this doesnt ICE
+
+int a;
+char b[8];
+int c() {
+ unsigned d = 0;
+ for (; d < a; ++d)
+ if (b[d + 1])
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
new file mode 100644
index 00000000000..db70c7ee87b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_12.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=neoverse-v3 -O3 --param=vect-hssr-usage=2" } */
+
+// Check this doesnt ICE
+
+char a;
+int b, d;
+char *c;
+void
+e ()
+{
+ int f;
+ while (f < d && c + f && (&a)[f] == c[f])
+ f++;
+ if (f)
+ b = d;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
new file mode 100644
index 00000000000..a6b49c3c9df
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_13.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=neoverse-v3 -O2 -mautovec-preference=sve-only
-mmax-vectorization --param=vect-hssr-usage=2" } */
+
+// Check this doesn't ICE
+
+int c[3];
+long *d;
+int e, f, g;
+void h() {
+ for (;;) {
+ for (int i = 0; i < g; i++)
+ if (c[f + i] || d[e])
+ break;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
new file mode 100644
index 00000000000..bba9893c3a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_14.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=neoverse-v3 -O2 -mautovec-preference=sve-only
-mmax-vectorization --param=vect-hssr-usage=2" } */
+
+// Check this doesn't ICE
+
+int a, d, e, f, g;
+int *b, *c;
+void h() {
+ int i, j = 0;
+ for (; a; j++) {
+ for (; g; g++)
+ f = b[j];
+ for (; i < e; i++)
+ if (c[i])
+ break;
+ d = i;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
new file mode 100644
index 00000000000..45198266cee
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_2.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+void foo(
+ type * const restrict dst1,
+ type * const restrict dst2,
+ type * const restrict src1,
+ type * const restrict src1b,
+ type * const restrict src2,
+ type * const restrict src3,
+ unsigned int n)
+{
+
+ for (int i = 0; i < n; i++) {
+ type v1 = src1[i];
+ type v1b = src1b[i];
+ if (v1 == v1b) {break;}
+ type v2 = src2[i];
+ dst1[i] = v1 + v2;
+ if (v2 == 1) {break;}
+ type v3 = src3[i];
+ dst2[i] = v1 + v2 + v3;
+ }
+}
+
+/* { dg-final { scan-tree-dump "Will use hssr" "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1h\tz[0-9]+\.h, p[0-9]+/z,} 3 } }
*/
+
+// We need two hssr regions here so there should be 2 reads
+
+/* { dg-final { scan-assembler-times {\trdffrs?\tp[0-9]+\.b} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
new file mode 100644
index 00000000000..b52869c99a3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_3.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+type call [[gnu::simd, gnu::const]] (type);
+
+void foo(
+ type * const restrict dst1,
+ type * const restrict src1,
+ type * const restrict src1b,
+ type * const restrict src2,
+ unsigned int n)
+{
+
+ for (int i = 0; i < n; i++) {
+ type v1b = call (src1b[i]);
+ type v1 = src1[i];
+ if (v1 == v1b) {break;}
+ type v2 = src2[i];
+ dst1[i] = v1 + v2;
+ }
+}
+
+/* { dg-final { scan-tree-dump "Will use hssr" "vect" } } */
+
+// Make sure the call does not come between the loads and the rdffr
+// As the call can clobber the HSSR state
+/*
+** foo:
+** ...
+** ldff1h z[0-9]+\.h, p[0-9]/z, \[x[0-9]+\]
+** ldff1h z[0-9]+\.h, p[0-9]/z, \[x[0-9]+\]
+** ...
+** rdffrs p[0-9]+.b, p[0-9]+/z
+** ...
+** b.nlast \.L[0-9]+
+** ...
+** bl _ZGVsMxv_call
+** ...
+*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
new file mode 100644
index 00000000000..9def8e348ec
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_4.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+ #include <stdint.h>
+void
+foo (uint16_t *const restrict dst1,
+ uint8_t *const restrict src1,
+ uint16_t *const restrict src2, unsigned int n)
+{
+ for (int i = 0; i < n && src1[2 * i] + src1[2 * i + 1] != 5; i++)
+ {
+ uint8_t v1 = src1[2 * i];
+ uint8_t v1a = src1[2 * i + 1];
+ uint16_t v2 = src2[i];
+ dst1[i] = v1 + v1a + v2;
+ }
+}
+
+/* This should not be vectorized with HSSR as we can't handle multiple lanes.
*/
+/* { dg-final { scan-tree-dump-not "Will use hssr" "vect" } } */
+/* { dg-final { scan-assembler-not {\tldff1} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
new file mode 100644
index 00000000000..e41c453188a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_5.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+
+#include <stdint.h>
+
+#define type uint16_t
+
+void
+foo (uint16_t *const restrict dst1,
+ uint16_t *const restrict src1,
+ uint16_t *const restrict src2,
+ unsigned int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ uint16_t v1 = src1[i];
+ uint16_t v2 = src1[i] == 0 ? 0 : src2[i];
+ if (v1 + v2 == 100) break;
+ dst1[i] = v1 + v2;
+ }
+}
+
+/* This should not be vectorized with HSSR as can not mask HSSR reads. */
+/* { dg-final { scan-tree-dump-not "Will use hssr" "vect" } } */
+/* { dg-final { scan-assembler-not {\tldff1} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
new file mode 100644
index 00000000000..4f225746be3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 --param=vect-hssr-usage=2" } */
+
+#include <stdint.h>
+
+__attribute__ ((noipa))
+int
+foo_vect (uint16_t *const restrict out,
+ uint16_t *const restrict src1,
+ uint16_t *const restrict src2,
+ unsigned int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ uint16_t v1 = src1[i];
+ uint16_t v2 = src2[i];
+ *out += v1 == v2;
+ if (v1 == 0)
+ return 1;
+ }
+ return 0;
+}
+
+__attribute__ ((noipa))
+int
+foo_no_vect (uint16_t *const restrict out,
+ uint16_t *const restrict src1,
+ uint16_t *const restrict src2,
+ unsigned int n)
+{
+#pragma GCC novector
+ for (int i = 0; i < n; i++)
+ {
+ uint16_t v1 = src1[i];
+ uint16_t v2 = src2[i];
+ *out += v1 == v2;
+ if (v1 == 0)
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-assembler-times {\tldff1} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
new file mode 100644
index 00000000000..452c7307cff
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_6_run.c
@@ -0,0 +1,82 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O3 --param=vect-hssr-usage=2" } */
+
+#include "ffr_6.c"
+#include <stdint.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#define N 10000
+int main () {
+ uint16_t *b1 = malloc (sizeof(uint16_t) * N);
+ uint16_t *b2 = malloc (sizeof(uint16_t) * N);
+
+ for (int i = 0; i < N; i++) {
+ b1[i] = 1 + (i % 3) * 3;
+ b2[i] = 1 + (i % 11) * 11;
+ /* These should only intersect when i % 7 = 0 and i % 11 = 0 */
+ }
+
+ uint16_t vect_out;
+ uint16_t no_vect_out;
+ int res_vect;
+ int res_no_vect;
+
+ /* Check while aligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1, b2, N);
+ res_no_vect = foo_no_vect(&no_vect_out, b1, b2, N);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while mutually misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+5, b2+5, N-5);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+5, b2+5, N-5);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while independently misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+9, b2+25, N-25);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+9, b2+25, N-25);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* insert an early break and check it still works */
+
+ b1[N/2+75] = 0;
+
+ /* Check while aligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1, b2, N);
+ res_no_vect = foo_no_vect(&no_vect_out, b1, b2, N);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while mutually misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+5, b2+5, N-5);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+5, b2+5, N-5);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+
+ /* Check while independently misaligned */
+ vect_out = 0;
+ no_vect_out = 0;
+ res_vect = foo_vect(&vect_out, b1+9, b2+25, N-25);
+ res_no_vect = foo_no_vect(&no_vect_out, b1+9, b2+25, N-25);
+
+ assert (res_vect == res_no_vect);
+ assert (vect_out == no_vect_out);
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
new file mode 100644
index 00000000000..1f508eb370e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_7.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps
-mtune=generic-armv9-a --param=vect-hssr-usage=2" } */
+// Options to try get the cost model to select VNx4SI
+
+#include <stdint.h>
+
+int foo (char *restrict a, char *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] + b[i] == 0)
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "Will use hssr" "vect" } } */
+/* Check it actually uses the widening loads as desired. */
+/* { dg-final { scan-assembler-times {\tldff1b\tz[0-9]+\.s, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
new file mode 100644
index 00000000000..bdecf2e1875
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_8.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fdump-tree-vect-details --save-temps -mtune=generic
--param=vect-hssr-usage=2" } */
+// Options to get the cost model to select VNx8HI
+
+#include <stdint.h>
+
+int foo (char *restrict a, char *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] + b[i] == 0)
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "Will use hssr" "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1b\tz[0-9]+\.h, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
b/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
new file mode 100644
index 00000000000..5b4a38355af
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ffr_9.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-vect-details --save-temps
--param=vect-hssr-usage=2" } */
+
+/* Check we use HSSR at O2. */
+
+#include <stdint.h>
+
+int foo (int *restrict a, int *restrict b, int N) {
+ for (int i = 0; i < N; i++)
+ {
+ if (a[i] == b[i])
+ return 1;
+ }
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "Will use hssr" "vect" } } */
+/* { dg-final { scan-assembler-times {\tldff1w\tz[0-9]+\.s, p[0-9]+/z,} 2 } }
*/
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index d7a815ebaec..28a5978d9a3 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -3212,6 +3212,23 @@ vect_get_loop_iv_increment (loop_vec_info loop_vinfo)
return iv_increment;
}
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ {
+ slp_instance instance;
+ int i;
+ FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i,
+ instance)
+ if (SLP_INSTANCE_HSSR_REGION (instance))
+ {
+ SLP_INSTANCE_HSSR_REGION (instance)->num_iter
+ = make_temp_ssa_name (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo),
+ NULL, "hssr_num_iters");
+ SSA_NAME_DEF_STMT (SLP_INSTANCE_HSSR_REGION (instance)->num_iter)
+ = gimple_build_nop ();
+ return SLP_INSTANCE_HSSR_REGION (instance)->num_iter;
+ }
+ gcc_unreachable ();
+ }
else
return build_int_cst (sizetype, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
}
@@ -4958,3 +4975,220 @@ vect_can_add_hssr_controls (loop_vec_info loop_vinfo)
return true;
}
+
+/* Insert the HSSR read. */
+void
+vect_add_hssr_read (vec_info *vinfo, slp_instance instance)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ gcc_assert (loop_vinfo);
+
+ gimple_stmt_iterator insert_point
+ = gsi_for_stmt (SLP_INSTANCE_HSSR_REGION (instance)->hssr_read_point);
+
+ gcall *hssr_read_call = gimple_build_call_internal (IFN_READ_HSSR_STATE, 0);
+ gsi_insert_after (&insert_point, hssr_read_call, GSI_SAME_STMT);
+
+ update_ssa (TODO_update_ssa_only_virtuals);
+
+ SLP_INSTANCE_HSSR_REGION (instance)->hssr_read_point = hssr_read_call;
+}
+
+/* Inserts fixup block for HSSR controls.
+
+ This inserts the following logic:
+
+ <bb 0>:
+ v1 = .MASK_FF_HSSR_LOAD (src1, 64B, loop_mask, { 0, ... });
+ v2 = .MASK_FF_HSSR_LOAD (src2, 64B, loop_mask, { 0, ... });
+ hssr_mask = .READ_FAULT_STATE ();
+ if (hssr_mask == { -1, ... })
+ goto <bb 2>; [99.95%]
+ else
+ goto <bb 1>; [0.05%]
+
+ <bb 1>:
+ .SET_FAULT_STATE ({ -1, ... });
+ hssr_loop_mask = loop_mask & hssr_mask;
+ num_iters = .COUNT_ACTIVE (hssr_mask);
+
+ <bb 2>:
+ # loop_mask_1 = PHI <loop_mask(0), hssr_loop_mask(1)>
+ # num_iters_56 = PHI <POLY_INT_CST [4, 4](0), num_iters (1)>
+
+ ... Logic using loop_mask_1 and incrementing IV's by num_iters_56
+ */
+void
+vect_add_hssr_fixup_controls (loop_vec_info loop_vinfo)
+{
+ gcc_assert (LOOP_VINFO_USING_HSSR_P (loop_vinfo));
+ gcc_assert (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
+ slp_instance instance;
+ int index;
+ FOR_EACH_VEC_ELT_REVERSE (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), index,
+ instance)
+ {
+ /* This instance does not define an HSSR region. No work to do. */
+ if (!SLP_INSTANCE_HSSR_REGION (instance))
+ continue;
+
+ /* At the very least the GCOND must use the HSSR mask so it can't be that
+ there are no uses. */
+ gcc_assert (!SLP_INSTANCE_HSSR_REGION (instance)->controls.is_empty ());
+
+ hssr_region *region = SLP_INSTANCE_HSSR_REGION (instance);
+ vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ rgroup_controls *base_rgm = &masks->rgc_vec[0];
+ tree mask_type = base_rgm->type;
+
+ /* The base mask is established by reading from hssr state. */
+ tree hssr_read_val = make_temp_ssa_name (mask_type, NULL, "hssr_mask");
+ gimple_call_set_lhs (region->hssr_read_point, hssr_read_val);
+
+ gcall *detect_fault_call
+ = gimple_build_call_internal (IFN_DETECT_HSSR_FAULT, 1,
+ hssr_read_val);
+ tree fault_detected_p = make_temp_ssa_name (boolean_type_node,
+ detect_fault_call,
+ "partial_read_p");
+ gimple_call_set_lhs (detect_fault_call, fault_detected_p);
+ gimple_stmt_iterator gsi_read = gsi_for_stmt (region->hssr_read_point);
+ gsi_insert_after (&gsi_read, detect_fault_call, GSI_NEW_STMT);
+
+ /* The fixup is taken if the load is partial. */
+ gcond *cond = gimple_build_cond (EQ_EXPR, fault_detected_p,
+ boolean_false_node, NULL_TREE,
+ NULL_TREE);
+ gsi_insert_after (&gsi_read, cond, GSI_SAME_STMT);
+
+ /* Split the BB after the FF reads and setup control flow for the fixup
+ block. */
+ basic_block read_bb = cond->bb;
+ edge non_hssr_edge = split_block (read_bb, cond);
+ non_hssr_edge->flags = EDGE_TRUE_VALUE;
+ non_hssr_edge->probability = profile_probability::very_likely ();
+
+ basic_block continuation_bb = non_hssr_edge->dest;
+ continuation_bb->count = read_bb->count;
+
+ basic_block fixup_bb = create_empty_bb (read_bb);
+ fixup_bb->count = read_bb->count.apply_scale (1, 100);
+
+ edge fixup_edge = make_edge (read_bb, fixup_bb, EDGE_FALSE_VALUE);
+ fixup_edge->probability = profile_probability::very_unlikely ();
+
+ edge fixup_cont_edge = make_edge (fixup_bb, continuation_bb,
+ EDGE_FALLTHRU);
+ fixup_cont_edge->probability = profile_probability::always ();
+
+ set_immediate_dominator (CDI_DOMINATORS, fixup_bb, read_bb);
+ set_immediate_dominator (CDI_DOMINATORS, continuation_bb, read_bb);
+
+ add_bb_to_loop (fixup_bb, LOOP_VINFO_LOOP (loop_vinfo));
+
+ /* Mask nodes in the fixup. */
+ vec<vec<tree>> temp_masks = {};
+ temp_masks.safe_grow_cleared (masks->rgc_vec.length ());
+ temp_masks[0].safe_grow (1);
+
+ gimple_seq fixup_logic = NULL;
+
+ /* We always need the loop to have the nV=1 mask. */
+ if (base_rgm->controls.is_empty ())
+ {
+ base_rgm->controls.safe_grow_cleared (1, true);
+ tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
+ /* Provide a dummy definition until the real one is available. */
+ base_rgm->controls[0] = mask;
+ }
+ if (region->controls[0].is_empty ())
+ {
+ region->controls[0].safe_grow_cleared (1, true);
+ region->controls[0][0]
+ = make_temp_ssa_name (mask_type, NULL, "loop_mask_AFTER_HSSR");
+ }
+ tree previous_mask = (region->prev_region)
+ ? region->prev_region->controls[0][0]
+ : masks->rgc_vec[0].controls[0];
+
+ gcall *call = gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
+ build_all_ones_cst (mask_type));
+ gimple_seq_add_stmt (&fixup_logic, call);
+
+ tree num_iters = gimple_build (&fixup_logic, IFN_COND_COUNT_ACTIVE,
+ sizetype, previous_mask, hssr_read_val,
+ build_zero_cst (mask_type));
+ temp_masks[0][0] = make_temp_ssa_name (mask_type, NULL,
"hssr_loop_mask");
+ gimple *hssr_and_loop
+ = gimple_build_assign (temp_masks[0][0], BIT_AND_EXPR, previous_mask,
+ hssr_read_val);
+ gimple_seq_add_stmt (&fixup_logic, hssr_and_loop);
+
+ /* PHI node for the NV=1 mask in the region. */
+ gphi *phi = create_phi_node (region->controls[0][0], continuation_bb);
+ add_phi_arg (phi, previous_mask, non_hssr_edge, UNKNOWN_LOCATION);
+ add_phi_arg (phi, temp_masks[0][0], fixup_cont_edge, UNKNOWN_LOCATION);
+
+ int i;
+ rgroup_controls *rgc;
+ FOR_EACH_VEC_ELT (masks->rgc_vec, i, rgc)
+ {
+ /* The base case is handled above. */
+ if (i == 0)
+ continue;
+
+ unsigned int nmasks = i + 1;
+ vec<tree> *this_masks = ®ion->controls[i];
+ vec<tree> *previous_masks = region->prev_region
+ ? ®ion->prev_region->controls[i]
+ : &rgc->controls;
+
+ vec<tree> *half_controls_temp = &(temp_masks[nmasks / 2 - 1]);
+ rgroup_controls *half_controls_rgc = &masks->rgc_vec[nmasks / 2 - 1];
+
+ vec<tree> *this_temps = &temp_masks[i];
+ this_temps->safe_grow_cleared (nmasks);
+ for (unsigned j = 0; j < nmasks; j++)
+ if (!(*this_temps)[j])
+ (*this_temps)[j] = make_temp_ssa_name (rgc->type, NULL,
+ "hssr_mask");
+
+ /* Permute the masks from the half rgoup to this one. */
+ gcc_assert (vect_maybe_permute_loop_masks (&fixup_logic, rgc,
+ half_controls_rgc,
+ this_temps,
+ half_controls_temp));
+
+ /* Create PHI nodes for the masks. */
+ for (unsigned int j = 0; j < nmasks; ++j)
+ {
+ gphi *mask_phi
+ = create_phi_node ((*this_masks)[j], continuation_bb);
+ add_phi_arg (mask_phi, (*previous_masks)[j], non_hssr_edge,
+ UNKNOWN_LOCATION);
+ add_phi_arg (mask_phi, (*this_temps)[j], fixup_cont_edge,
+ UNKNOWN_LOCATION);
+ }
+ }
+
+ /* Initialize the num_iter nodes. */
+ if (!region->num_iter)
+ region->num_iter
+ = make_temp_ssa_name (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo), NULL,
+ "hssr_num_iters");
+
+ gphi *num_iter_phi = create_phi_node (region->num_iter, continuation_bb);
+ add_phi_arg (num_iter_phi,
+ region->prev_region
+ ? region->prev_region->num_iter
+ : build_int_cst (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo),
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo)),
+ non_hssr_edge, UNKNOWN_LOCATION);
+ add_phi_arg (num_iter_phi,
+ num_iters,
+ fixup_cont_edge, UNKNOWN_LOCATION);
+
+ gimple_stmt_iterator fixup_gsi = gsi_start_bb (fixup_bb);
+ gsi_insert_seq_before (&fixup_gsi, fixup_logic, GSI_SAME_STMT);
+ }
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index b54cdf35aed..aa55c846719 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -60,6 +60,7 @@ along with GCC; see the file COPYING3. If not see
#include "langhooks.h"
#include "opts.h"
#include "hierarchical_discriminator.h"
+#include "tree-into-ssa.h"
/* Loop Vectorization Pass.
@@ -10932,7 +10933,7 @@ tree
vect_get_loop_mask (loop_vec_info loop_vinfo,
gimple_stmt_iterator *gsi, vec_loop_masks *masks,
unsigned int nvectors, tree vectype, unsigned int index,
- slp_tree slp_node ATTRIBUTE_UNUSED)
+ slp_tree slp_node)
{
if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
== vect_partial_vectors_while_ult)
@@ -10955,6 +10956,38 @@ vect_get_loop_mask (loop_vec_info loop_vinfo,
}
tree mask = rgm->controls[index];
+
+ /* If we are in an HSSR region, also populate all the other required
+ masks, and use the HSSR mask. */
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ {
+ for (slp_instance instance : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
+ if (hssr_region *region = SLP_INSTANCE_HSSR_REGION (instance))
+ {
+ if (region->controls.length () < nvectors)
+ region->controls.safe_grow_cleared (nvectors);
+
+ vec<tree> *controls = ®ion->controls[nvectors - 1];
+ if (controls->is_empty ())
+ {
+ controls->safe_grow_cleared (nvectors, true);
+ for (unsigned int i = 0; i < nvectors; ++i)
+ {
+ tree mask = make_temp_ssa_name (mask_type, NULL,
+ "hssr_loop_mask");
+ /* Provide a dummy definition until the real one is
+ available. */
+ SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
+ (*controls)[i] = mask;
+ }
+ }
+ }
+
+ if (SLP_TREE_HSSR_REGION (slp_node))
+ mask
+ = SLP_TREE_HSSR_REGION (slp_node)->controls[nvectors - 1][index];
+ }
+
if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
TYPE_VECTOR_SUBPARTS (vectype)))
{
@@ -11834,7 +11867,8 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
/* Both peeling for alignment and peeling for gaps can end up
with the scalar epilogue running for more than VF-1 iterations. */
&& !main_vinfo->peeling_for_alignment
- && !main_vinfo->peeling_for_gaps)
+ && !main_vinfo->peeling_for_gaps
+ && LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
{
unsigned int bound;
poly_uint64 main_iters
@@ -11850,7 +11884,10 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
= wi::umin ((bound_wide_int) (bound - 1),
loop->nb_iterations_upper_bound);
}
- }
+
+ if (loop_vinfo && !LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo))
+ loop->any_upper_bound = false;
+ }
if (loop->any_likely_upper_bound)
loop->nb_iterations_likely_upper_bound
= (final_iter_may_be_partial
@@ -11895,6 +11932,36 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple
*loop_vectorized_call)
" variable-length vectorization factor\n");
}
+ /* Save and restore the HSSR state. */
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo))
+ {
+ tree hssr_type = LOOP_VINFO_MASKS (loop_vinfo).rgc_vec[0].type;
+
+ tree hssr_ssa = make_temp_ssa_name (hssr_type, NULL,
"hssr_preservation");
+
+ gcall *call = gimple_build_call_internal (IFN_READ_HSSR_STATE, 0);
+ gimple_set_lhs (call, hssr_ssa);
+ gsi_insert_on_edge_immediate (loop_preheader_edge (loop), call);
+
+
+ /* Initializze the hssr state to all true. */
+ gcall *set_hssr_call
+ = gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
+ build_all_ones_cst (hssr_type));
+ gsi_insert_on_edge_immediate (loop_preheader_edge (loop), set_hssr_call);
+
+ auto_vec<edge> exits = get_loop_exit_edges (loop);
+ for (edge e: exits)
+ {
+ gcall *call = gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
+ hssr_ssa);
+ gsi_insert_on_edge_immediate (e, call);
+ }
+
+ vect_add_hssr_fixup_controls (loop_vinfo);
+ update_ssa (TODO_update_ssa_only_virtuals);
+ }
+
/* When we have unrolled the loop due to a user requested value we should
leave it up to the RTL unroll heuristics to determine if it's still worth
while to unroll more. */
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 47d8bef38da..bf63fb51d99 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -12019,6 +12019,13 @@ vect_schedule_slp_node (vec_info *vinfo,
gcc_checking_assert (!last_stmt
|| !is_ctrl_altering_stmt (last_stmt));
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+ if (loop_vinfo && LOOP_VINFO_USING_HSSR_P (loop_vinfo)
+ && SLP_TREE_HSSR_REGION (node))
+ if (vect_stmt_dominates_stmt_p (
+ last_stmt, SLP_TREE_HSSR_REGION (node)->hssr_read_point))
+ last_stmt = SLP_TREE_HSSR_REGION (node)->hssr_read_point;
+
if (is_a <bb_vec_info> (vinfo)
&& !SLP_TREE_PERMUTE_P (node)
&& (!last_stmt
@@ -12484,10 +12491,20 @@ vect_schedule_slp (vec_info *vinfo, const
vec<slp_instance> &slp_instances)
slp_instance instance;
unsigned int i;
+ loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
+
hash_map<slp_tree, slp_scc_info> scc_info;
int maxdfs = 0;
FOR_EACH_VEC_ELT (slp_instances, i, instance)
{
+ /* Insert the call to read the HSSR now as we need it for data
+ dependencies when scheduling, the actual fixup logic will be added
+ later. */
+ if (loop_vinfo
+ && LOOP_VINFO_USING_HSSR_P (loop_vinfo)
+ && SLP_INSTANCE_HSSR_REGION (instance))
+ vect_add_hssr_read (vinfo, instance);
+
slp_tree node = SLP_INSTANCE_TREE (instance);
if (dump_enabled_p ())
{
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index b77688b6f97..37734b162c5 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -58,6 +58,7 @@ along with GCC; see the file COPYING3. If not see
#include "attribs.h"
#include "optabs-libfuncs.h"
#include "tree-dfa.h"
+#include "tree-into-ssa.h"
/* For lang_hooks.types.type_for_mode. */
#include "langhooks.h"
@@ -1302,7 +1303,9 @@ vect_finish_replace_stmt (vec_info *vinfo,
}
/* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
- before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
+ before *GSI. Create and return a stmt_vec_info for VEC_STMT.
+
+ If this is a HSSR, insert the new load at the HSSR insertion point. */
void
vect_finish_stmt_generation (vec_info *vinfo,
@@ -11607,18 +11610,43 @@ vectorizable_load (vec_info *vinfo,
}
else if (final_mask)
{
- tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
- vec_els = vect_get_mask_load_else (maskload_elsval, vectype);
- if (type_mode_padding_p
- && maskload_elsval != MASK_LOAD_ELSE_ZERO)
- need_zeroing = true;
- gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
- dataref_ptr, ptr,
- final_mask,
- vec_els);
- gimple_call_set_nothrow (call, true);
- new_stmt = call;
- data_ref = NULL_TREE;
+ if (LOOP_VINFO_USING_HSSR_P (loop_vinfo)
+ && dr_safe_speculative_read_required (stmt_info)
+ && ls.defines_region)
+ {
+ /* Do the HSSR read, and then read the HSSR and save it for
+ this r-group. There should only be one, so we should be
+ safe... */
+ tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+ vec_els = vect_get_mask_load_else (maskload_elsval,
+ vectype);
+ if (type_mode_padding_p
+ && maskload_elsval != MASK_LOAD_ELSE_ZERO)
+ need_zeroing = true;
+ gcall *call
+ = gimple_build_call_internal (IFN_MASK_FF_HSSR_LOAD, 4,
+ dataref_ptr, ptr,
+ final_mask, vec_els);
+ gimple_call_set_nothrow (call, true);
+ new_stmt = call;
+ data_ref = NULL_TREE;
+ }
+ else
+ {
+ tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
+ vec_els = vect_get_mask_load_else (maskload_elsval,
+ vectype);
+ if (type_mode_padding_p
+ && maskload_elsval != MASK_LOAD_ELSE_ZERO)
+ need_zeroing = true;
+ gcall *call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
+ dataref_ptr, ptr,
+ final_mask,
+ vec_els);
+ gimple_call_set_nothrow (call, true);
+ new_stmt = call;
+ data_ref = NULL_TREE;
+ }
}
else
{
@@ -11965,7 +11993,7 @@ vectorizable_load (vec_info *vinfo,
/* Store vector loads in the corresponding SLP_NODE. */
if (!costing_p && !ls.slp_perm)
- slp_node->push_vec_def (new_stmt);
+ slp_node->push_vec_def (new_stmt, true);
/* With SLP permutation we load the gaps as well, without
we need to skip the gaps after we manage to fully load
@@ -12043,6 +12071,11 @@ vectorizable_load (vec_info *vinfo,
slp_node->data = new vect_load_store_data (std::move (ls));
}
+ /* If we used a HSSR on this, we need to update the virtual ssa nodes because
+ of the HSSR read. */
+ if (loop_vinfo && LOOP_VINFO_USING_HSSR_P (loop_vinfo)
+ && dr_safe_speculative_read_required (stmt_info))
+ update_ssa (TODO_update_ssa_only_virtuals);
return true;
}
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index a52e3dae738..b8297d95e5d 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1360,7 +1360,7 @@ public:
#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
#define LOOP_VINFO_IV_INCREMENT(L) (L)->iv_increment
#define LOOP_VINFO_IV_INCREMENT_INVARIANT_P(L) \
- (!LOOP_VINFO_USING_SELECT_VL_P (L))
+ (!LOOP_VINFO_USING_SELECT_VL_P (L) && !LOOP_VINFO_USING_HSSR_P (L))
#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
#define LOOP_VINFO_MASKS(L) (L)->masks
#define LOOP_VINFO_LENS(L) (L)->lens
@@ -2557,6 +2557,8 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class
loop *, edge,
bool = true);
class loop *vect_loop_versioning (loop_vec_info, gimple *);
extern bool vect_can_add_hssr_controls (loop_vec_info);
+extern void vect_add_hssr_fixup_controls (loop_vec_info);
+extern void vect_add_hssr_read (vec_info *, slp_instance);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
tree *, tree *, tree *, int, bool, bool,
tree *);
--
2.43.0
