When the partial loop control check is within the loop body, use the
current control instead of the next loop control for the latch.
gcc/ChangeLog:
PR tree-optimization/126301
* tree-vect-loop-manip.cc (vect_set_loop_controls_directly):
Change to use current iteration control when
LOOP_VINFO_EARLY_BREAKS_VECT_PEELED.
(vect_set_loop_condition_partial_vectors_avx512):
Change to use current iteration control when
LOOP_VINFO_EARLY_BREAKS_VECT_PEELED.
-- >8 --
Fixes the linked PR but has the undesirable side effect of separating
the check and branch from the mask definitions via branches so they can
no longer be combined.
I am working on a pass for that separately.
Reg tested and bootstrapped on aarch64, x86_64, armhf
KR,
Alfie
---
gcc/tree-vect-loop-manip.cc | 22 ++++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index ec9dec1b713..c69aea41ad3 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -511,6 +511,10 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
bool use_masks_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
+ /* Check if this control will be the main exit for the loop. */
+ bool main_exit_p = single_pred (LOOP_VINFO_LOOP (loop_vinfo)->latch)
+ == LOOP_VINFO_MAIN_EXIT (loop_vinfo)->src;
+
tree ctrl_type = rgc->type;
unsigned int nitems_per_iter = rgc->max_nscalars_per_iter * rgc->factor;
poly_uint64 nitems_per_ctrl = TYPE_VECTOR_SUBPARTS (ctrl_type) * rgc->factor;
@@ -714,7 +718,8 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
/* Provide a definition of each control in the group. */
tree next_ctrl = NULL_TREE;
- tree ctrl;
+ tree ctrl = NULL_TREE;
+ tree exit_ctrl = NULL_TREE;
unsigned int i;
FOR_EACH_VEC_ELT_REVERSE (rgc->controls, i, ctrl)
{
@@ -822,6 +827,10 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
gsi_insert_seq_before (test_gsi, seq, GSI_SAME_STMT);
}
+ /* If this is the main exit for the loop, then this will be in the header
+ of the loop and so we should check next_ctrl. Otherwise, this is
+ in the body of the loop, and so we should use ctrl. */
+ exit_ctrl = main_exit_p ? next_ctrl : ctrl;
vect_set_loop_control (loop, ctrl, init_ctrl, next_ctrl);
}
@@ -837,7 +846,7 @@ vect_set_loop_controls_directly (class loop *loop,
loop_vec_info loop_vinfo,
gimple_seq_add_stmt (header_seq, minus);
}
- return next_ctrl;
+ return exit_ctrl;
}
/* Set up the iteration condition and rgroup controls for LOOP, given
@@ -1023,6 +1032,10 @@ vect_set_loop_condition_partial_vectors_avx512 (class
loop *loop,
tree orig_niters = niters;
gimple_seq preheader_seq = NULL;
+ /* Check if this control will be the main exit for the loop. */
+ bool main_exit_p = single_pred (LOOP_VINFO_LOOP (loop_vinfo)->latch)
+ == LOOP_VINFO_MAIN_EXIT (loop_vinfo)->src;
+
/* Create an IV that counts down from niters and whose step
is the number of iterations processed in the current iteration.
Produce the controls with compares like the following.
@@ -1113,6 +1126,7 @@ vect_set_loop_condition_partial_vectors_avx512 (class
loop *loop,
tree next_ctrl = NULL_TREE;
tree first_rem = NULL_TREE;
tree ctrl;
+ tree exit_ctrl = NULL_TREE;
unsigned int i;
FOR_EACH_VEC_ELT_REVERSE (rgc.controls, i, ctrl)
{
@@ -1201,6 +1215,10 @@ vect_set_loop_condition_partial_vectors_avx512 (class
loop *loop,
UNKNOWN_LOCATION,
LT_EXPR, ctrl_type, cmp_series, rem);
+ /* If this is the main exit for the loop, then this will be in the
+ header of the loop and so we should use next_ctrl. Otherwise,
+ this is in the body of the loop, and so we should use ctrl. */
+ exit_ctrl = main_exit_p ? next_ctrl : ctrl;
vect_set_loop_control (loop, ctrl, init_ctrl, next_ctrl);
}
}
--
2.34.1
