I was incorrect previously thinking PR 126301 only affected partial
vectors.
This duplicates Tamars fix for the other two control flow branches.
PR tree-optimization/126301
gcc/ChangeLog:
* tree-vect-loop-manip.cc (vect_use_loop_latch_condition_p):
Remove restriction for avx512 and non-partial vectors.
(vect_set_loop_condition_partial_vectors_avx512):
Add logic for do loop transformation.
(vect_set_loop_condition_normal): Likewise.
Co-Authored-By: Tamar Christina <[email protected]>
---
gcc/tree-vect-loop-manip.cc | 92 +++++++++++++++++++++++++++++++------
1 file changed, 77 insertions(+), 15 deletions(-)
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 385353bda2b..f8e259c19bd 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -520,10 +520,7 @@ static bool
vect_use_loop_latch_condition_p (loop_vec_info loop_vinfo)
{
return (loop_vinfo
- && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)
- && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)
- && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo)
- != vect_partial_vectors_avx512));
+ && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo));
}
/* Helper for vect_set_loop_condition_partial_vectors. Generate definitions
@@ -1347,7 +1344,36 @@ vect_set_loop_condition_partial_vectors_avx512 (class
loop *loop,
iv_type. */
gcond *cond_stmt = gimple_build_cond (code, index_before_incr, iv_step,
NULL_TREE, NULL_TREE);
- gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT);
+ edge latch_exit_edge = NULL;
+ if (vect_use_loop_latch_condition_p (loop_vinfo))
+ {
+ basic_block latch = loop->latch;
+ edge latch_e = single_succ_edge (latch);
+ int exit_flags = exit_edge->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
+
+ latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
+ latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags;
+ latch_exit_edge = make_edge (latch, exit_edge->dest, exit_flags);
+ latch_exit_edge->probability = exit_edge->probability;
+ latch_exit_edge->count () = exit_edge->count ();
+ copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge);
+ if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest))
+ SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge,
+ get_live_virtual_operand_on_edge
+ (latch_exit_edge));
+ gimple_stmt_iterator latch_gsi = gsi_last_bb (latch);
+ gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT);
+ LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge;
+
+ gcond *old_cond = as_a <gcond *> (gsi_stmt (loop_cond_gsi));
+ if (exit_edge->flags & EDGE_TRUE_VALUE)
+ gimple_cond_make_false (old_cond);
+ else
+ gimple_cond_make_true (old_cond);
+ update_stmt (old_cond);
+ }
+ else
+ gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT);
/* The loop iterates (NITERS - 1 + NITERS_SKIP) / VF + 1 times.
Subtract one from this to get the latch count. */
@@ -1485,16 +1511,43 @@ vect_set_loop_condition_normal (loop_vec_info
loop_vinfo, edge exit_edge,
&indx_before_incr, &indx_after_incr,
!loop_vinfo || LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
- indx_after_incr = force_gimple_operand_gsi (&loop_cond_gsi, indx_after_incr,
- true, NULL_TREE, true,
- GSI_SAME_STMT);
- limit = force_gimple_operand_gsi (&loop_cond_gsi, limit, true, NULL_TREE,
- true, GSI_SAME_STMT);
-
cond_stmt = gimple_build_cond (code, indx_after_incr, limit, NULL_TREE,
NULL_TREE);
- gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT);
+ edge latch_exit_edge = NULL;
+ if (vect_use_loop_latch_condition_p (loop_vinfo))
+ {
+ basic_block latch = loop->latch;
+ edge latch_e = single_succ_edge (latch);
+ int exit_flags = exit_edge->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
+
+ latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
+ latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags;
+ latch_exit_edge = make_edge (latch, exit_edge->dest, exit_flags);
+ latch_exit_edge->probability = exit_edge->probability;
+ latch_exit_edge->count () = exit_edge->count ();
+ copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge);
+ if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest))
+ SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge,
+ get_live_virtual_operand_on_edge
+ (latch_exit_edge));
+ gimple_stmt_iterator latch_gsi = gsi_last_bb (latch);
+ gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT);
+ LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge;
+
+ gcond *old_cond = as_a <gcond *> (gsi_stmt (loop_cond_gsi));
+ if (exit_edge->flags & EDGE_TRUE_VALUE)
+ gimple_cond_make_false (old_cond);
+ else
+ gimple_cond_make_true (old_cond);
+ update_stmt (old_cond);
+ }
+ else
+ {
+ limit = force_gimple_operand_gsi (&loop_cond_gsi, limit, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+ gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT);
+ }
/* Record the number of latch iterations. */
if (limit == niters)
@@ -1511,7 +1564,6 @@ vect_set_loop_condition_normal (loop_vec_info loop_vinfo,
edge exit_edge,
if (final_iv)
{
gassign *assign;
- gcc_assert (single_pred_p (exit_edge->dest));
tree phi_dest
= integer_zerop (init) ? final_iv : copy_ssa_name (indx_after_incr);
/* Make sure to maintain LC SSA form here and elide the subtraction
@@ -1520,8 +1572,18 @@ vect_set_loop_condition_normal (loop_vec_info
loop_vinfo, edge exit_edge,
add_phi_arg (phi, indx_after_incr, exit_edge, UNKNOWN_LOCATION);
if (!integer_zerop (init))
{
- assign = gimple_build_assign (final_iv, MINUS_EXPR,
- phi_dest, init);
+ /* If vectorizing an inverted early break loop we have to restart the
+ scalar loop at niters - vf. This matches what we do in
+ vect_gen_vector_loop_niters_mult_vf for non-masked loops. */
+ if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
+ {
+ tree ftype = TREE_TYPE (final_iv);
+ tree vf
+ = build_int_cst (ftype, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+ assign = gimple_build_assign (final_iv, MINUS_EXPR, init, vf);
+ }
+ else
+ assign = gimple_build_assign (final_iv, MINUS_EXPR, phi_dest, init);
gimple_stmt_iterator gsi = gsi_after_labels (exit_edge->dest);
gsi_insert_before (&gsi, assign, GSI_SAME_STMT);
}
--
2.55.0
