On Fri, 9 Oct 2026, Alfie Richards wrote: > I was incorrect previously thinking PR 126301 only affected partial > vectors. > > This duplicates Tamars fix for the other two control flow branches.
OK. Thanks, Richard. > PR tree-optimization/126301 > > gcc/ChangeLog: > > * tree-vect-loop-manip.cc (vect_use_loop_latch_condition_p): > Remove restriction for avx512 and non-partial vectors. > (vect_reuse_loop_latch): New function. > (vect_set_loop_condition_partial_vectors_avx512): > Add logic for do loop transformation. > (vect_set_loop_condition_normal): Likewise. > > Co-Authored-By: Tamar Christina <[email protected]> > > -- >8 -- > > Hi Richi, > > Thanks for the feedback, it definitely benefited from factoring that out. > > Bootstrapped and regression tested again on armhv, aarch64, x86_64 > > Okay for master? > > Thanks, > Alfie > > --- > gcc/tree-vect-loop-manip.cc | 157 +++++++++++++++++++----------------- > 1 file changed, 84 insertions(+), 73 deletions(-) > > diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc > index 385353bda2b..714257f498d 100644 > --- a/gcc/tree-vect-loop-manip.cc > +++ b/gcc/tree-vect-loop-manip.cc > @@ -519,11 +519,7 @@ get_live_virtual_operand_on_edge (edge e) > static bool > vect_use_loop_latch_condition_p (loop_vec_info loop_vinfo) > { > - return (loop_vinfo > - && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo) > - && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) > - && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) > - != vect_partial_vectors_avx512)); > + return (loop_vinfo && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)); > } > > /* Helper for vect_set_loop_condition_partial_vectors. Generate definitions > @@ -909,6 +905,65 @@ vect_set_loop_controls_directly (class loop *loop, > loop_vec_info loop_vinfo, > return next_ctrl; > } > > + /* Convert the loop into a do-while form similar to what ch_vect would have > + done. We know that after the checks and peeling that we have at least > one > + iteration to perform of the loop because the loop is PEELED. A PEELED > loop > + has the increment exit before the early ones, i.e. it's a do-while loop > but > + if we materialize the IV edge in that place we are essentially checking > one > + iteration ahead so we exit early. Instead when using masks and the loop > + is PEELED we remove the existing loop latch and make it a fall through > + edge and place the latch back to the end of the loop. So effectively > + transform: > + > + header > + | > + latch > + | > + body > + | > + branch to header > + > + into > + > + header > + | > + body > + | > + newlatch > + | > + branch to header > + > + because the conditions in the pre-header makes it safe to do so for some > + cases. */ > +static void > +vect_reuse_loop_latch (loop_vec_info loop_vinfo, basic_block latch, > + edge exit_edge, gcond *old_cond, gcond *cond_stmt) > +{ > + edge latch_e = single_succ_edge (latch); > + int exit_flags = exit_edge->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); > + edge latch_exit_edge = NULL; > + > + latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); > + latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags; > + latch_exit_edge = make_edge (latch, exit_edge->dest, exit_flags); > + latch_exit_edge->probability = exit_edge->probability; > + latch_exit_edge->count () = exit_edge->count (); > + copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge); > + if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest)) > + SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge, > + get_live_virtual_operand_on_edge ( > + latch_exit_edge)); > + gimple_stmt_iterator latch_gsi = gsi_last_bb (latch); > + gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT); > + LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge; > + > + if (exit_edge->flags & EDGE_TRUE_VALUE) > + gimple_cond_make_false (old_cond); > + else > + gimple_cond_make_true (old_cond); > + update_stmt (old_cond); > +} > + > /* Set up the iteration condition and rgroup controls for LOOP, given > that LOOP_VINFO_USING_PARTIAL_VECTORS_P is true for the vectorized > loop. LOOP_VINFO describes the vectorization of LOOP. NITERS is > @@ -1037,64 +1092,9 @@ vect_set_loop_condition_partial_vectors (class loop > *loop, edge exit_edge, > cond_stmt > = gimple_build_cond (code, test_ctrl, zero_ctrl, NULL_TREE, NULL_TREE); > } > - edge latch_exit_edge = NULL; > - /* Convert the loop into a do-while form similar to what ch_vect would have > - done. We know that after the checks and peeling that we have at least > one > - iteration to perform of the loop because the loop is PEELED. A PEELED > loop > - has the increment exit before the early ones, i.e. it's a do-while loop > but > - if we materialize the IV edge in that place we are essentially checking > one > - iteration ahead so we exit early. Instead when using masks and the loop > - is PEELED we remove the existing loop latch and make it a fall through > - edge and place the latch back to the end of the loop. So effectively > - transform: > - > - header > - | > - latch > - | > - body > - | > - branch to header > - > - into > - > - header > - | > - body > - | > - newlatch > - | > - branch to header > - > - because the conditions in the pre-header makes it safe to do so for some > - cases. */ > if (vect_use_loop_latch_condition_p (loop_vinfo)) > - { > - basic_block latch = loop->latch; > - edge latch_e = single_succ_edge (latch); > - int exit_flags = exit_edge->flags & (EDGE_TRUE_VALUE | > EDGE_FALSE_VALUE); > - > - latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | > EDGE_FALSE_VALUE); > - latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags; > - latch_exit_edge = make_edge (latch, exit_edge->dest, exit_flags); > - latch_exit_edge->probability = exit_edge->probability; > - latch_exit_edge->count () = exit_edge->count (); > - copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge); > - if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest)) > - SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge, > - get_live_virtual_operand_on_edge > - (latch_exit_edge)); > - gimple_stmt_iterator latch_gsi = gsi_last_bb (latch); > - gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT); > - LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge; > - > - gcond *old_cond = as_a <gcond *> (gsi_stmt (loop_cond_gsi)); > - if (exit_edge->flags & EDGE_TRUE_VALUE) > - gimple_cond_make_false (old_cond); > - else > - gimple_cond_make_true (old_cond); > - update_stmt (old_cond); > - } > + vect_reuse_loop_latch (loop_vinfo, loop->latch, exit_edge, > + as_a<gcond *> (gsi_stmt (loop_cond_gsi)), cond_stmt); > else > gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > > @@ -1347,7 +1347,11 @@ vect_set_loop_condition_partial_vectors_avx512 (class > loop *loop, > iv_type. */ > gcond *cond_stmt = gimple_build_cond (code, index_before_incr, iv_step, > NULL_TREE, NULL_TREE); > - gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > + if (vect_use_loop_latch_condition_p (loop_vinfo)) > + vect_reuse_loop_latch (loop_vinfo, loop->latch, exit_edge, > + as_a<gcond *> (gsi_stmt (loop_cond_gsi)), cond_stmt); > + else > + gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > > /* The loop iterates (NITERS - 1 + NITERS_SKIP) / VF + 1 times. > Subtract one from this to get the latch count. */ > @@ -1485,16 +1489,14 @@ vect_set_loop_condition_normal (loop_vec_info > loop_vinfo, edge exit_edge, > &indx_before_incr, &indx_after_incr, > !loop_vinfo || LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)); > > - indx_after_incr = force_gimple_operand_gsi (&loop_cond_gsi, > indx_after_incr, > - true, NULL_TREE, true, > - GSI_SAME_STMT); > - limit = force_gimple_operand_gsi (&loop_cond_gsi, limit, true, NULL_TREE, > - true, GSI_SAME_STMT); > - > cond_stmt = gimple_build_cond (code, indx_after_incr, limit, NULL_TREE, > NULL_TREE); > > - gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > + if (vect_use_loop_latch_condition_p (loop_vinfo)) > + vect_reuse_loop_latch (loop_vinfo, loop->latch, exit_edge, > + as_a<gcond *> (gsi_stmt (loop_cond_gsi)), cond_stmt); > + else > + gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > > /* Record the number of latch iterations. */ > if (limit == niters) > @@ -1511,7 +1513,6 @@ vect_set_loop_condition_normal (loop_vec_info > loop_vinfo, edge exit_edge, > if (final_iv) > { > gassign *assign; > - gcc_assert (single_pred_p (exit_edge->dest)); > tree phi_dest > = integer_zerop (init) ? final_iv : copy_ssa_name (indx_after_incr); > /* Make sure to maintain LC SSA form here and elide the subtraction > @@ -1520,8 +1521,18 @@ vect_set_loop_condition_normal (loop_vec_info > loop_vinfo, edge exit_edge, > add_phi_arg (phi, indx_after_incr, exit_edge, UNKNOWN_LOCATION); > if (!integer_zerop (init)) > { > - assign = gimple_build_assign (final_iv, MINUS_EXPR, > - phi_dest, init); > + /* If vectorizing an inverted early break loop we have to restart the > + scalar loop at niters - vf. This matches what we do in > + vect_gen_vector_loop_niters_mult_vf for non-masked loops. */ > + if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) > + { > + tree ftype = TREE_TYPE (final_iv); > + tree vf > + = build_int_cst (ftype, LOOP_VINFO_VECT_FACTOR (loop_vinfo)); > + assign = gimple_build_assign (final_iv, MINUS_EXPR, init, vf); > + } > + else > + assign = gimple_build_assign (final_iv, MINUS_EXPR, phi_dest, init); > gimple_stmt_iterator gsi = gsi_after_labels (exit_edge->dest); > gsi_insert_before (&gsi, assign, GSI_SAME_STMT); > } > > base-commit: b47bddeb20c80f44a0e8cf849b557d7c2e26eb78 > -- Richard Biener <[email protected]> SUSE Software Solutions Germany GmbH, Frankenstrasse 146, 90461 Nuernberg, Germany; GF: Stefan Gaiser, Jochen Jaser, Abhinav Puri; (HRB 36809, AG Nuernberg)
