The loop header rewrite in vect_set_loop_condition_partial_vectors runs after
early break store movement and so it has to update any virtual operands on the
exit edge.
Bootstrapped Regtested on aarch64-none-linux-gnu, arm-none-linux-gnueabihf,
x86_64-pc-linux-gnu -m32, -m64 and no issues.
Pushed.
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/126778
* tree-vect-loop-manip.cc (get_live_virtual_operand_on_edge): Move it.
(vect_set_loop_condition_partial_vectors): Use it to update edge.
gcc/testsuite/ChangeLog:
PR tree-optimization/126778
* gcc.dg/vect/vect-early-break_145-pr126778_1.c: New test.
* gcc.dg/vect/vect-early-break_145-pr126778_2.c: New test.
---
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c
b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c
new file mode 100644
index
0000000000000000000000000000000000000000..e49710367cf1ad6027d007617408788d99266a6b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c
@@ -0,0 +1,16 @@
+/* PR tree-optimization/126778 */
+/* { dg-do compile } */
+/* { dg-add-options vect_early_break } */
+/* { dg-require-effective-target vect_early_break } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+/* { dg-additional-options "-march=armv8-a+sve" { target aarch64*-*-* } } */
+
+void
+glob3 (char *sc, short *dc)
+{
+ while (dc && (*dc++ = *sc))
+ ;
+}
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target aarch64*-*-*
} } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c
b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c
new file mode 100644
index
0000000000000000000000000000000000000000..67092a1d37f4ebb397a845948d89881f0d564bea
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c
@@ -0,0 +1,45 @@
+/* PR tree-optimization/126778 */
+/* { dg-do run } */
+/* { dg-add-options vect_early_break } */
+/* { dg-require-effective-target vect_early_break_hw } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target aarch64_sve_hw { target aarch64*-*-* } } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+/* { dg-additional-options "-march=armv8-a+sve" { target aarch64*-*-* } } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+glob3 (char *sc, short *dc)
+{
+ while (dc && (*dc++ = *sc))
+ ;
+}
+
+int
+main (void)
+{
+ check_vect ();
+
+ char c = 0;
+ short dst[16];
+
+#pragma GCC novector
+ for (int i = 0; i < 16; ++i)
+ dst[i] = 42;
+
+ glob3 (&c, dst);
+
+ if (dst[0] != 0)
+ __builtin_abort ();
+
+#pragma GCC novector
+ for (int i = 1; i < 16; ++i)
+ if (dst[i] != 42)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target aarch64*-*-*
} } } */
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index
96c10340efd94e7db503af208ca21cf3fd807dcd..b54353399d9ed6454442c93750b42924513178f5
100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -464,6 +464,32 @@ vect_iv_increment_position (edge loop_exit,
gimple_stmt_iterator *bsi,
*insert_after = false;
}
+/* Get the virtual operand live on E. The precondition on this is valid
+ immediate dominators and an actual virtual definition dominating E. */
+/* ??? Costly band-aid. For the use in question we can populate a
+ live-on-exit/end-of-BB virtual operand when copying stmts. */
+
+static tree
+get_live_virtual_operand_on_edge (edge e)
+{
+ basic_block bb = e->src;
+ do
+ {
+ for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ if (gimple_vdef (stmt))
+ return gimple_vdef (stmt);
+ if (gimple_vuse (stmt))
+ return gimple_vuse (stmt);
+ }
+ if (gphi *vphi = get_virtual_phi (bb))
+ return gimple_phi_result (vphi);
+ bb = get_immediate_dominator (CDI_DOMINATORS, bb);
+ }
+ while (1);
+}
+
/* If this is a loop where the latch condition should be rewritten to reflect
a control flow change from a while-do to a do-while loop. */
@@ -1031,6 +1057,10 @@ vect_set_loop_condition_partial_vectors (class loop
*loop, edge exit_edge,
latch_exit_edge->probability = exit_edge->probability;
latch_exit_edge->count () = exit_edge->count ();
copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge);
+ if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest))
+ SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge,
+ get_live_virtual_operand_on_edge
+ (latch_exit_edge));
gimple_stmt_iterator latch_gsi = gsi_last_bb (latch);
gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT);
LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge;
@@ -1559,32 +1589,6 @@ vect_set_loop_condition (class loop *loop, edge loop_e,
loop_vec_info loop_vinfo
(gimple *) cond_stmt);
}
-/* Get the virtual operand live on E. The precondition on this is valid
- immediate dominators and an actual virtual definition dominating E. */
-/* ??? Costly band-aid. For the use in question we can populate a
- live-on-exit/end-of-BB virtual operand when copying stmts. */
-
-static tree
-get_live_virtual_operand_on_edge (edge e)
-{
- basic_block bb = e->src;
- do
- {
- for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
- {
- gimple *stmt = gsi_stmt (gsi);
- if (gimple_vdef (stmt))
- return gimple_vdef (stmt);
- if (gimple_vuse (stmt))
- return gimple_vuse (stmt);
- }
- if (gphi *vphi = get_virtual_phi (bb))
- return gimple_phi_result (vphi);
- bb = get_immediate_dominator (CDI_DOMINATORS, bb);
- }
- while (1);
-}
-
/* Given LOOP this function generates a new copy of it and puts it
on E which is either the entry or exit of LOOP. If SCALAR_LOOP is
non-NULL, assume LOOP and SCALAR_LOOP are equivalent and copy the
--
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c
new file mode 100644
index 0000000000000000000000000000000000000000..e49710367cf1ad6027d007617408788d99266a6b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_1.c
@@ -0,0 +1,16 @@
+/* PR tree-optimization/126778 */
+/* { dg-do compile } */
+/* { dg-add-options vect_early_break } */
+/* { dg-require-effective-target vect_early_break } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+/* { dg-additional-options "-march=armv8-a+sve" { target aarch64*-*-* } } */
+
+void
+glob3 (char *sc, short *dc)
+{
+ while (dc && (*dc++ = *sc))
+ ;
+}
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target aarch64*-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c
new file mode 100644
index 0000000000000000000000000000000000000000..67092a1d37f4ebb397a845948d89881f0d564bea
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_145-pr126778_2.c
@@ -0,0 +1,45 @@
+/* PR tree-optimization/126778 */
+/* { dg-do run } */
+/* { dg-add-options vect_early_break } */
+/* { dg-require-effective-target vect_early_break_hw } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target aarch64_sve_hw { target aarch64*-*-* } } */
+/* { dg-additional-options "-O3 -fno-vect-cost-model" } */
+/* { dg-additional-options "-march=armv8-a+sve" { target aarch64*-*-* } } */
+
+#include "tree-vect.h"
+
+__attribute__ ((noipa))
+void
+glob3 (char *sc, short *dc)
+{
+ while (dc && (*dc++ = *sc))
+ ;
+}
+
+int
+main (void)
+{
+ check_vect ();
+
+ char c = 0;
+ short dst[16];
+
+#pragma GCC novector
+ for (int i = 0; i < 16; ++i)
+ dst[i] = 42;
+
+ glob3 (&c, dst);
+
+ if (dst[0] != 0)
+ __builtin_abort ();
+
+#pragma GCC novector
+ for (int i = 1; i < 16; ++i)
+ if (dst[i] != 42)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target aarch64*-*-* } } } */
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 96c10340efd94e7db503af208ca21cf3fd807dcd..b54353399d9ed6454442c93750b42924513178f5 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -464,6 +464,32 @@ vect_iv_increment_position (edge loop_exit, gimple_stmt_iterator *bsi,
*insert_after = false;
}
+/* Get the virtual operand live on E. The precondition on this is valid
+ immediate dominators and an actual virtual definition dominating E. */
+/* ??? Costly band-aid. For the use in question we can populate a
+ live-on-exit/end-of-BB virtual operand when copying stmts. */
+
+static tree
+get_live_virtual_operand_on_edge (edge e)
+{
+ basic_block bb = e->src;
+ do
+ {
+ for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ if (gimple_vdef (stmt))
+ return gimple_vdef (stmt);
+ if (gimple_vuse (stmt))
+ return gimple_vuse (stmt);
+ }
+ if (gphi *vphi = get_virtual_phi (bb))
+ return gimple_phi_result (vphi);
+ bb = get_immediate_dominator (CDI_DOMINATORS, bb);
+ }
+ while (1);
+}
+
/* If this is a loop where the latch condition should be rewritten to reflect
a control flow change from a while-do to a do-while loop. */
@@ -1031,6 +1057,10 @@ vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge,
latch_exit_edge->probability = exit_edge->probability;
latch_exit_edge->count () = exit_edge->count ();
copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge);
+ if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest))
+ SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge,
+ get_live_virtual_operand_on_edge
+ (latch_exit_edge));
gimple_stmt_iterator latch_gsi = gsi_last_bb (latch);
gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT);
LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge;
@@ -1559,32 +1589,6 @@ vect_set_loop_condition (class loop *loop, edge loop_e, loop_vec_info loop_vinfo
(gimple *) cond_stmt);
}
-/* Get the virtual operand live on E. The precondition on this is valid
- immediate dominators and an actual virtual definition dominating E. */
-/* ??? Costly band-aid. For the use in question we can populate a
- live-on-exit/end-of-BB virtual operand when copying stmts. */
-
-static tree
-get_live_virtual_operand_on_edge (edge e)
-{
- basic_block bb = e->src;
- do
- {
- for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
- {
- gimple *stmt = gsi_stmt (gsi);
- if (gimple_vdef (stmt))
- return gimple_vdef (stmt);
- if (gimple_vuse (stmt))
- return gimple_vuse (stmt);
- }
- if (gphi *vphi = get_virtual_phi (bb))
- return gimple_phi_result (vphi);
- bb = get_immediate_dominator (CDI_DOMINATORS, bb);
- }
- while (1);
-}
-
/* Given LOOP this function generates a new copy of it and puts it
on E which is either the entry or exit of LOOP. If SCALAR_LOOP is
non-NULL, assume LOOP and SCALAR_LOOP are equivalent and copy the
