On Thu, 8 Oct 2026, Alfie Richards wrote: > I was incorrect previously thinking PR 126301 only affected partial > vectors. > > This duplicates Tamars fix for the other two control flow branches.
I wonder if it is possible to factor the boiler-plate out? I suppose there's another copy on the SVE path? > PR tree-optimization/126301 > > gcc/ChangeLog: > > * tree-vect-loop-manip.cc (vect_use_loop_latch_condition_p): > Remove restriction for avx512 and non-partial vectors. > (vect_set_loop_condition_partial_vectors_avx512): > Add logic for do loop transformation. > (vect_set_loop_condition_normal): Likewise. > > Co-Authored-By: Tamar Christina <[email protected]> > --- > gcc/tree-vect-loop-manip.cc | 92 +++++++++++++++++++++++++++++++------ > 1 file changed, 77 insertions(+), 15 deletions(-) > > diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc > index 385353bda2b..f8e259c19bd 100644 > --- a/gcc/tree-vect-loop-manip.cc > +++ b/gcc/tree-vect-loop-manip.cc > @@ -520,10 +520,7 @@ static bool > vect_use_loop_latch_condition_p (loop_vec_info loop_vinfo) > { > return (loop_vinfo > - && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo) > - && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) > - && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) > - != vect_partial_vectors_avx512)); > + && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)); > } > > /* Helper for vect_set_loop_condition_partial_vectors. Generate definitions > @@ -1347,7 +1344,36 @@ vect_set_loop_condition_partial_vectors_avx512 (class > loop *loop, > iv_type. */ > gcond *cond_stmt = gimple_build_cond (code, index_before_incr, iv_step, > NULL_TREE, NULL_TREE); > - gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > + edge latch_exit_edge = NULL; > + if (vect_use_loop_latch_condition_p (loop_vinfo)) > + { > + basic_block latch = loop->latch; > + edge latch_e = single_succ_edge (latch); > + int exit_flags = exit_edge->flags & (EDGE_TRUE_VALUE | > EDGE_FALSE_VALUE); > + > + latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | > EDGE_FALSE_VALUE); > + latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags; > + latch_exit_edge = make_edge (latch, exit_edge->dest, exit_flags); > + latch_exit_edge->probability = exit_edge->probability; > + latch_exit_edge->count () = exit_edge->count (); > + copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge); > + if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest)) > + SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge, > + get_live_virtual_operand_on_edge > + (latch_exit_edge)); > + gimple_stmt_iterator latch_gsi = gsi_last_bb (latch); > + gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT); > + LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge; > + > + gcond *old_cond = as_a <gcond *> (gsi_stmt (loop_cond_gsi)); > + if (exit_edge->flags & EDGE_TRUE_VALUE) > + gimple_cond_make_false (old_cond); > + else > + gimple_cond_make_true (old_cond); > + update_stmt (old_cond); > + } > + else > + gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > > /* The loop iterates (NITERS - 1 + NITERS_SKIP) / VF + 1 times. > Subtract one from this to get the latch count. */ > @@ -1485,16 +1511,43 @@ vect_set_loop_condition_normal (loop_vec_info > loop_vinfo, edge exit_edge, > &indx_before_incr, &indx_after_incr, > !loop_vinfo || LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo)); > > - indx_after_incr = force_gimple_operand_gsi (&loop_cond_gsi, > indx_after_incr, > - true, NULL_TREE, true, > - GSI_SAME_STMT); > - limit = force_gimple_operand_gsi (&loop_cond_gsi, limit, true, NULL_TREE, > - true, GSI_SAME_STMT); > - > cond_stmt = gimple_build_cond (code, indx_after_incr, limit, NULL_TREE, > NULL_TREE); > > - gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > + edge latch_exit_edge = NULL; > + if (vect_use_loop_latch_condition_p (loop_vinfo)) > + { > + basic_block latch = loop->latch; > + edge latch_e = single_succ_edge (latch); > + int exit_flags = exit_edge->flags & (EDGE_TRUE_VALUE | > EDGE_FALSE_VALUE); > + > + latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | > EDGE_FALSE_VALUE); > + latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags; > + latch_exit_edge = make_edge (latch, exit_edge->dest, exit_flags); > + latch_exit_edge->probability = exit_edge->probability; > + latch_exit_edge->count () = exit_edge->count (); > + copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge); > + if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest)) > + SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge, > + get_live_virtual_operand_on_edge > + (latch_exit_edge)); > + gimple_stmt_iterator latch_gsi = gsi_last_bb (latch); > + gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT); > + LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge; > + > + gcond *old_cond = as_a <gcond *> (gsi_stmt (loop_cond_gsi)); > + if (exit_edge->flags & EDGE_TRUE_VALUE) > + gimple_cond_make_false (old_cond); > + else > + gimple_cond_make_true (old_cond); > + update_stmt (old_cond); > + } > + else > + { > + limit = force_gimple_operand_gsi (&loop_cond_gsi, limit, true, > NULL_TREE, > + true, GSI_SAME_STMT); > + gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); > + } > > /* Record the number of latch iterations. */ > if (limit == niters) > @@ -1511,7 +1564,6 @@ vect_set_loop_condition_normal (loop_vec_info > loop_vinfo, edge exit_edge, > if (final_iv) > { > gassign *assign; > - gcc_assert (single_pred_p (exit_edge->dest)); > tree phi_dest > = integer_zerop (init) ? final_iv : copy_ssa_name (indx_after_incr); > /* Make sure to maintain LC SSA form here and elide the subtraction > @@ -1520,8 +1572,18 @@ vect_set_loop_condition_normal (loop_vec_info > loop_vinfo, edge exit_edge, > add_phi_arg (phi, indx_after_incr, exit_edge, UNKNOWN_LOCATION); > if (!integer_zerop (init)) > { > - assign = gimple_build_assign (final_iv, MINUS_EXPR, > - phi_dest, init); > + /* If vectorizing an inverted early break loop we have to restart the > + scalar loop at niters - vf. This matches what we do in > + vect_gen_vector_loop_niters_mult_vf for non-masked loops. */ > + if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) > + { > + tree ftype = TREE_TYPE (final_iv); > + tree vf > + = build_int_cst (ftype, LOOP_VINFO_VECT_FACTOR (loop_vinfo)); > + assign = gimple_build_assign (final_iv, MINUS_EXPR, init, vf); > + } > + else > + assign = gimple_build_assign (final_iv, MINUS_EXPR, phi_dest, init); > gimple_stmt_iterator gsi = gsi_after_labels (exit_edge->dest); > gsi_insert_before (&gsi, assign, GSI_SAME_STMT); > } > -- Richard Biener <[email protected]> SUSE Software Solutions Germany GmbH, Frankenstrasse 146, 90461 Nuernberg, Germany; GF: Stefan Gaiser, Jochen Jaser, Abhinav Puri; (HRB 36809, AG Nuernberg)
