https://gcc.gnu.org/g:1b7a552c8dbb712244968ac528481d0cf12d3461
commit r17-5740-g1b7a552c8dbb712244968ac528481d0cf12d3461 Author: Tamar Christina <[email protected]> Date: Thu Oct 29 13:14:34 2026 +0000 vect: Fix peeled early-break loop counts [PR128028] This patch moves the counting-exit handling for peeled early-break loops into vect_set_loop_condition. The loop-control helpers now build the condition and leave the caller to place it. This shares the code for full and partial vectors and removes vect_reuse_loop_latch. It also fixes the iteration counts for loops where the counting exit comes before the body. For example: int bump (const long *restrict a, const long *restrict b, long *restrict c, unsigned long n) { unsigned long i = 0; while (i != n) { unsigned long j = i++; c[j] += 1; if (a[j] != b[j]) return 1; } return 0; } Previously the code would adjust the IV and the bounds during loop peeling and would set it up in anticipating of the latch of the loop bein in an unnatural position. After the patch of mine to peel the header the latch is now in a normal position. Which means that by updating the IV to be that of the previous iteration we would do the final iteration twice. The tests now tests for this as well. Conceptually by moving the header adjustment code into the caller, i.e. vect_gen_vector_loop_niters, we can remove almost all the special treatment of LOOP_VINFO_EARLY_BREAKS_VECT_PEELED from the loop manip code, since we know loop control would always adjust it. By doing so we can also remove all the additional code added to the individual loop control methods that conditional adjust where the latch is inserted. The caller now decides and so the code is much simpler. Only uncounted loops are treated special still, and we might be able to remove that as well. But I'm delaying that until I see Victor's patches to update uncounted loops. Co-Authored-By: Alfie Richards <[email protected]> gcc/ChangeLog: PR tree-optimization/128028 * tree-vect-loop-manip.cc (vect_set_loop_controls_directly): Move the trip count adjustment to vect_gen_vector_loop_niters. (vect_reuse_loop_latch): Remove. (vect_set_loop_condition_partial_vectors): Leave condition insertion and final IV updates to the caller. (vect_set_loop_condition_partial_vectors_avx512): Likewise. (vect_set_loop_condition_normal): Leave condition insertion to the caller and use the completed iteration count. (vect_set_loop_condition): Share exit relocation and partial-vector final IV updates. (vect_gen_vector_loop_niters): Exclude the header-only iteration. (vect_gen_vector_loop_niters_mult_vf): Remove the VF subtraction. (vect_do_peeling): Replay the main exit only for uncounted loops. * tree-vect-loop.cc (vect_analyze_loop_2): Account for the header-only iteration in the versioning threshold. gcc/testsuite/ChangeLog: PR tree-optimization/128028 * gcc.dg/vect/vect-early-break-pr128028-unlimited.c: New test. * gcc.dg/vect/vect-early-break-pr128028.c: New test. Diff: --- .../vect/vect-early-break-pr128028-unlimited.c | 6 + .../gcc.dg/vect/vect-early-break-pr128028.c | 146 ++++++++++++ gcc/tree-vect-loop-manip.cc | 248 ++++++--------------- gcc/tree-vect-loop.cc | 4 +- 4 files changed, 225 insertions(+), 179 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break-pr128028-unlimited.c b/gcc/testsuite/gcc.dg/vect/vect-early-break-pr128028-unlimited.c new file mode 100644 index 000000000000..c33161cfe497 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-early-break-pr128028-unlimited.c @@ -0,0 +1,6 @@ +/* { dg-do run } */ +/* { dg-add-options vect_early_break } */ +/* { dg-require-effective-target vect_early_break_hw } */ +/* { dg-additional-options "-O3 -fvect-cost-model=unlimited" } */ + +#include "vect-early-break-pr128028.c" diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break-pr128028.c b/gcc/testsuite/gcc.dg/vect/vect-early-break-pr128028.c new file mode 100644 index 000000000000..b9e5a9b579cc --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-early-break-pr128028.c @@ -0,0 +1,146 @@ +/* { dg-do run } */ +/* { dg-add-options vect_early_break } */ +/* { dg-require-effective-target vect_early_break_hw } */ +/* { dg-additional-options "-O3" } */ + +#define N 64 + +/* The counting exit is tested before the early exit, so the loop body runs + one time less than the loop header. The vector loop must not process a + lane for the iteration that only tests the counter, as that lane reads the + element next to the arrays. */ + +__attribute__ ((noipa)) int +differs (const long *a, const long *b, unsigned long n) +{ + while (n != 0) + { + n--; + if (a[n] != b[n]) + return 1; + } + return 0; +} + +__attribute__ ((noipa)) int +differs_signed (const long *a, const long *b, long n) +{ + if (n < 0) + return 2; + while (--n >= 0) + { + if (a[n] != b[n]) + return 1; + } + return 0; +} + +__attribute__ ((noipa)) int +differs_up (const long *a, const long *b, unsigned long n) +{ + unsigned long i = 0; + while (i != n) + { + unsigned long j = i; + i++; + if (a[j] != b[j]) + return 1; + } + return 0; +} + +/* The store before the early exit must not write the extra lane and must run + once for every element. */ + +__attribute__ ((noipa)) int +bump (const long *restrict a, const long *restrict b, long *restrict c, + unsigned long n) +{ + while (n != 0) + { + n--; + c[n] += 1; + if (a[n] != b[n]) + return 1; + } + return 0; +} + +/* On the normal exit, the scalar epilogue must not repeat the stores from + the final vector iteration. */ + +__attribute__ ((noipa)) int +bump_up (const long *restrict a, const long *restrict b, long *restrict c, + unsigned long n) +{ + unsigned long i = 0; + while (i != n) + { + unsigned long j = i++; + c[j] += 1; + if (a[j] != b[j]) + return 1; + } + return 0; +} + +static long abuf[N + 16] __attribute__ ((aligned (64))); +static long bbuf[N + 16] __attribute__ ((aligned (64))); +static long cbuf[N + 16] __attribute__ ((aligned (64))); + +int +main (void) +{ + for (unsigned long n = 0; n <= N; n++) + for (int off = 0; off < 8; off++) + { + /* Start the arrays at every alignment of a 64 byte vector and give + the elements around them different values. */ + long *a = abuf + 8 + off; + long *b = bbuf + 8 + off; + long *c = cbuf + 8 + off; + for (unsigned long i = 0; i < n; i++) + a[i] = b[i] = c[i] = i * 3 + 1; + a[-1] = 1; + b[-1] = 2; + c[-1] = 5; + a[n] = 3; + b[n] = 4; + c[n] = 6; + + if (differs (a, b, n) != 0 || differs_signed (a, b, n) != 0 + || differs_up (a, b, n) != 0) + __builtin_abort (); + + if (bump (a, b, c, n) != 0 || c[-1] != 5 || c[n] != 6) + __builtin_abort (); + for (unsigned long i = 0; i < n; i++) + if (c[i] != i * 3 + 2) + __builtin_abort (); + + for (unsigned long i = 0; i < n; i++) + c[i] = i * 3 + 1; + if (bump_up (a, b, c, n) != 0 || c[-1] != 5 || c[n] != 6) + __builtin_abort (); + for (unsigned long i = 0; i < n; i++) + if (c[i] != i * 3 + 2) + __builtin_abort (); + + if (n > 0) + { + b[n / 2]++; + if (differs (a, b, n) != 1 || differs_signed (a, b, n) != 1 + || differs_up (a, b, n) != 1) + __builtin_abort (); + for (unsigned long i = 0; i < n; i++) + c[i] = i * 3 + 1; + if (bump_up (a, b, c, n) != 1 || c[-1] != 5 || c[n] != 6) + __builtin_abort (); + for (unsigned long i = 0; i < n; i++) + if (c[i] != i * 3 + 1 + (i <= n / 2)) + __builtin_abort (); + } + } + + return 0; +} diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc index 56b2a557cde7..e1a1c5c1dd1c 100644 --- a/gcc/tree-vect-loop-manip.cc +++ b/gcc/tree-vect-loop-manip.cc @@ -789,13 +789,6 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, bias_tree); } - /* A do-while loop always executes the body once, as such the limit - the end counter should be lowered by 1 iteration. */ - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - this_test_limit = gimple_build (preheader_seq, MINUS_EXPR, - compare_type, this_test_limit, - build_one_cst (compare_type)); - /* Create the initial control. First include all items that are within the loop limit. */ tree init_ctrl = NULL_TREE; @@ -892,79 +885,18 @@ vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, return next_ctrl; } - /* Convert the loop into a do-while form similar to what ch_vect would have - done. We know that after the checks and peeling that we have at least one - iteration to perform of the loop because the loop is PEELED. A PEELED loop - has the increment exit before the early ones, i.e. it's a do-while loop but - if we materialize the IV edge in that place we are essentially checking one - iteration ahead so we exit early. Instead when using masks and the loop - is PEELED we remove the existing loop latch and make it a fall through - edge and place the latch back to the end of the loop. So effectively - transform: - - header - | - latch - | - body - | - branch to header - - into - - header - | - body - | - newlatch - | - branch to header - - because the conditions in the pre-header makes it safe to do so for some - cases. */ -static void -vect_reuse_loop_latch (loop_vec_info loop_vinfo, basic_block latch, - edge exit_edge, gcond *old_cond, gcond *cond_stmt) -{ - edge latch_e = single_succ_edge (latch); - int exit_flags = exit_edge->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); - edge latch_exit_edge = NULL; - - latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); - latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags; - latch_exit_edge = make_edge (latch, exit_edge->dest, exit_flags); - latch_exit_edge->probability = exit_edge->probability; - latch_exit_edge->count () = exit_edge->count (); - copy_phi_arg_into_existing_phi (exit_edge, latch_exit_edge); - if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest)) - SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge, - get_live_virtual_operand_on_edge ( - latch_exit_edge)); - gimple_stmt_iterator latch_gsi = gsi_last_bb (latch); - gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT); - LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge; - - if (exit_edge->flags & EDGE_TRUE_VALUE) - gimple_cond_make_false (old_cond); - else - gimple_cond_make_true (old_cond); - update_stmt (old_cond); -} - /* Set up the iteration condition and rgroup controls for LOOP, given that LOOP_VINFO_USING_PARTIAL_VECTORS_P is true for the vectorized loop. LOOP_VINFO describes the vectorization of LOOP. NITERS is the number of iterations of the original scalar loop that should be - handled by the vector loop. NITERS_MAYBE_ZERO and FINAL_IV are as - for vect_set_loop_condition. - - Insert the branch-back condition before LOOP_COND_GSI and return the - final gcond. */ + handled by the vector loop. NITERS_MAYBE_ZERO is as for + vect_set_loop_condition. Return the branch-back condition for the caller + to insert. */ static gcond * vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge, loop_vec_info loop_vinfo, tree niters, - tree final_iv, bool niters_maybe_zero, + bool niters_maybe_zero, gimple_stmt_iterator loop_cond_gsi) { gimple_seq preheader_seq = NULL; @@ -973,7 +905,6 @@ vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge, bool use_masks_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo); tree compare_type = LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo); unsigned int compare_precision = TYPE_PRECISION (compare_type); - tree orig_niters = niters; /* Type of the initial value of NITERS. */ tree ni_actual_type = TREE_TYPE (niters); @@ -1079,12 +1010,6 @@ vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge, cond_stmt = gimple_build_cond (code, test_ctrl, zero_ctrl, NULL_TREE, NULL_TREE); } - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - vect_reuse_loop_latch (loop_vinfo, loop->latch, exit_edge, - as_a<gcond *> (gsi_stmt (loop_cond_gsi)), cond_stmt); - else - gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); - /* The loop iterates (NITERS - 1) / VF + 1 times. Subtract one from this to get the latch count. */ tree step = build_int_cst (compare_type, @@ -1094,23 +1019,6 @@ vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge, loop->nb_iterations = fold_build2 (TRUNC_DIV_EXPR, compare_type, niters_minus_one, step); - if (final_iv) - { - gassign *assign; - /* If vectorizing an inverted early break loop we have to restart the - scalar loop at niters - vf. This matches what we do in - vect_gen_vector_loop_niters_mult_vf for non-masked loops. */ - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - { - tree ftype = TREE_TYPE (orig_niters); - tree vf = build_int_cst (ftype, LOOP_VINFO_VECT_FACTOR (loop_vinfo)); - assign = gimple_build_assign (final_iv, MINUS_EXPR, orig_niters, vf); - } - else - assign = gimple_build_assign (final_iv, orig_niters); - gsi_insert_on_edge_immediate (LOOP_VINFO_MAIN_EXIT (loop_vinfo), assign); - } - return cond_stmt; } @@ -1118,19 +1026,15 @@ vect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge, style, given that LOOP_VINFO_USING_PARTIAL_VECTORS_P is true for the vectorized loop. LOOP_VINFO describes the vectorization of LOOP. NITERS is the number of iterations of the original scalar loop that should be - handled by the vector loop. NITERS_MAYBE_ZERO and FINAL_IV are as - for vect_set_loop_condition. - - Insert the branch-back condition before LOOP_COND_GSI and return the - final gcond. */ + handled by the vector loop. NITERS_MAYBE_ZERO is as for + vect_set_loop_condition. Return the branch-back condition for the caller + to insert. */ static gcond * vect_set_loop_condition_partial_vectors_avx512 (class loop *loop, edge exit_edge, loop_vec_info loop_vinfo, tree niters, - tree final_iv, - bool niters_maybe_zero, - gimple_stmt_iterator loop_cond_gsi) + bool niters_maybe_zero) { tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo); tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo); @@ -1334,12 +1238,6 @@ vect_set_loop_condition_partial_vectors_avx512 (class loop *loop, iv_type. */ gcond *cond_stmt = gimple_build_cond (code, index_before_incr, iv_step, NULL_TREE, NULL_TREE); - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - vect_reuse_loop_latch (loop_vinfo, loop->latch, exit_edge, - as_a<gcond *> (gsi_stmt (loop_cond_gsi)), cond_stmt); - else - gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); - /* The loop iterates (NITERS - 1 + NITERS_SKIP) / VF + 1 times. Subtract one from this to get the latch count. */ tree niters_minus_one @@ -1352,49 +1250,27 @@ vect_set_loop_condition_partial_vectors_avx512 (class loop *loop, loop->nb_iterations = fold_build2 (TRUNC_DIV_EXPR, iv_type, niters_adj2, iv_step); - if (final_iv) - { - gassign *assign; - /* If vectorizing an inverted early break loop we have to restart the - scalar loop at niters - vf. This matches what we do in - vect_gen_vector_loop_niters_mult_vf for non-masked loops. */ - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - { - tree ftype = TREE_TYPE (orig_niters); - tree vf = build_int_cst (ftype, LOOP_VINFO_VECT_FACTOR (loop_vinfo)); - assign = gimple_build_assign (final_iv, MINUS_EXPR, orig_niters, vf); - } - else - assign = gimple_build_assign (final_iv, orig_niters); - gsi_insert_on_edge_immediate (LOOP_VINFO_MAIN_EXIT (loop_vinfo), assign); - } - return cond_stmt; } /* Like vect_set_loop_condition, but handle the case in which the vector - loop handles exactly VF scalars per iteration. */ + loop handles exactly VF scalars per iteration. Return the branch-back + condition for the caller to insert. */ static gcond * vect_set_loop_condition_normal (loop_vec_info loop_vinfo, edge exit_edge, class loop *loop, tree niters, tree step, - tree final_iv, bool niters_maybe_zero, - gimple_stmt_iterator loop_cond_gsi) + tree final_iv, bool niters_maybe_zero) { tree indx_before_incr, indx_after_incr; gcond *cond_stmt; - gcond *orig_cond; edge pe = loop_preheader_edge (loop); gimple_stmt_iterator incr_gsi; bool insert_after; enum tree_code code; tree niters_type = TREE_TYPE (niters); - orig_cond = get_loop_exit_condition (exit_edge); - gcc_assert (orig_cond); - loop_cond_gsi = gsi_for_stmt (orig_cond); - tree init, limit; if (!niters_maybe_zero && integer_onep (step)) { @@ -1479,12 +1355,6 @@ vect_set_loop_condition_normal (loop_vec_info loop_vinfo, edge exit_edge, cond_stmt = gimple_build_cond (code, indx_after_incr, limit, NULL_TREE, NULL_TREE); - if (loop_vinfo && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - vect_reuse_loop_latch (loop_vinfo, loop->latch, exit_edge, - as_a<gcond *> (gsi_stmt (loop_cond_gsi)), cond_stmt); - else - gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); - /* Record the number of latch iterations. */ if (limit == niters) /* Case A: the loop iterates NITERS times. Subtract one to get the @@ -1508,18 +1378,7 @@ vect_set_loop_condition_normal (loop_vec_info loop_vinfo, edge exit_edge, add_phi_arg (phi, indx_after_incr, exit_edge, UNKNOWN_LOCATION); if (!integer_zerop (init)) { - /* If vectorizing an inverted early break loop we have to restart the - scalar loop at niters - vf. This matches what we do in - vect_gen_vector_loop_niters_mult_vf for non-masked loops. */ - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - { - tree ftype = TREE_TYPE (final_iv); - tree vf - = build_int_cst (ftype, LOOP_VINFO_VECT_FACTOR (loop_vinfo)); - assign = gimple_build_assign (final_iv, MINUS_EXPR, init, vf); - } - else - assign = gimple_build_assign (final_iv, MINUS_EXPR, phi_dest, init); + assign = gimple_build_assign (final_iv, MINUS_EXPR, phi_dest, init); gimple_stmt_iterator gsi = gsi_after_labels (LOOP_VINFO_MAIN_EXIT (loop_vinfo)->dest); gsi_insert_before (&gsi, assign, GSI_SAME_STMT); @@ -1578,13 +1437,12 @@ vect_set_loop_condition (class loop *loop, edge loop_e, loop_vec_info loop_vinfo if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo) == vect_partial_vectors_avx512) cond_stmt = vect_set_loop_condition_partial_vectors_avx512 (loop, loop_e, loop_vinfo, - niters, final_iv, - niters_maybe_zero, - loop_cond_gsi); + niters, + niters_maybe_zero); else cond_stmt = vect_set_loop_condition_partial_vectors (loop, loop_e, loop_vinfo, - niters, final_iv, + niters, niters_maybe_zero, loop_cond_gsi); } @@ -1592,13 +1450,43 @@ vect_set_loop_condition (class loop *loop, edge loop_e, loop_vec_info loop_vinfo cond_stmt = vect_set_loop_condition_normal (loop_vinfo, loop_e, loop, niters, step, final_iv, - niters_maybe_zero, - loop_cond_gsi); + niters_maybe_zero); - /* Remove old loop exit test. */ - stmt_vec_info orig_cond_info; - if (!loop_vinfo || !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) + /* A peeled early-break loop is entered only when it has at least one + vector iteration to execute. Test the IV after that iteration rather + than before it, regardless of the style of loop control. */ + if (loop_vinfo && LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) { + /* Move the IV exit test from the header to the latch, so that it + follows the early exits in the loop body. */ + basic_block latch = loop->latch; + edge latch_e = single_succ_edge (latch); + int exit_flags = loop_e->flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); + + latch_e->flags &= ~(EDGE_FALLTHRU | EDGE_TRUE_VALUE | EDGE_FALSE_VALUE); + latch_e->flags |= (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE) ^ exit_flags; + edge latch_exit_edge = make_edge (latch, loop_e->dest, exit_flags); + latch_exit_edge->probability = loop_e->probability; + latch_exit_edge->count () = loop_e->count (); + copy_phi_arg_into_existing_phi (loop_e, latch_exit_edge); + if (gphi *vphi = get_virtual_phi (latch_exit_edge->dest)) + SET_PHI_ARG_DEF_ON_EDGE (vphi, latch_exit_edge, + get_live_virtual_operand_on_edge + (latch_exit_edge)); + gimple_stmt_iterator latch_gsi = gsi_last_bb (latch); + gsi_insert_after (&latch_gsi, cond_stmt, GSI_NEW_STMT); + LOOP_VINFO_MAIN_EXIT (loop_vinfo) = latch_exit_edge; + + if (loop_e->flags & EDGE_TRUE_VALUE) + gimple_cond_make_false (orig_cond); + else + gimple_cond_make_true (orig_cond); + update_stmt (orig_cond); + } + else + { + gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT); + stmt_vec_info orig_cond_info; if (loop_vinfo && (orig_cond_info = loop_vinfo->lookup_stmt (orig_cond))) loop_vinfo->remove_stmt (orig_cond_info); @@ -1606,6 +1494,13 @@ vect_set_loop_condition (class loop *loop, edge loop_e, loop_vec_info loop_vinfo gsi_remove (&loop_cond_gsi, true); } + if (loop_vinfo && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo) + && final_iv) + { + gassign *assign = gimple_build_assign (final_iv, niters); + gsi_insert_on_edge_immediate (LOOP_VINFO_MAIN_EXIT (loop_vinfo), assign); + } + if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "New loop exit condition: %G", (gimple *) cond_stmt); @@ -3108,6 +3003,13 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters, poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo)); + /* With a counting exit before the loop body, the final scalar iteration + only tests that exit. Exclude it from both the vector trip count and + the masks, since the vector loop tests its bound after executing the + body. */ + if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) + niters = fold_build2 (MINUS_EXPR, type, niters, build_one_cst (type)); + /* If epilogue loop is required because of data accesses with gaps, we subtract one iteration from the total number of iterations here for correct calculation of RATIO. */ @@ -3250,12 +3152,6 @@ vect_gen_vector_loop_niters_mult_vf (loop_vec_info loop_vinfo, tree niters_vector_mult_vf = fold_build2 (MULT_EXPR, type, niters_vector, tree_vf); - /* If we've peeled a vector iteration then subtract one full vector - iteration. */ - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) - niters_vector_mult_vf = fold_build2 (MINUS_EXPR, type, - niters_vector_mult_vf, tree_vf); - if (!is_gimple_val (niters_vector_mult_vf)) { tree var = create_tmp_var (type, "niters_vector_mult_vf"); @@ -3951,10 +3847,9 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, scale_loop_profile (epilog, prob_epilog, -1); } - /* If we have a peeled vector iteration, all exits are the same, leave it - and so the main exit needs to be treated the same as the alternative - exits in that we leave their updates to vectorizable_live_operations. - */ + /* A counted main exit follows the completed vector iteration. + Uncounted loops must replay the exiting iteration to find the + scalar exit. */ tree vector_iters_vf = niters_vector_mult_vf; if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)) { @@ -3984,17 +3879,14 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, vect_update_ivs_after_vectorizer (loop_vinfo, tmp_niters_vf, update_e, true); } - if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) + if (uncounted_p) vector_iters_vf = tmp_niters_vf; LOOP_VINFO_EARLY_BRK_NITERS_VAR (loop_vinfo) = tmp_niters_vf; } - bool recalculate_peel_niters_init - = LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo); - vect_update_ivs_after_vectorizer (loop_vinfo, vector_iters_vf, - update_e, - recalculate_peel_niters_init); + vect_update_ivs_after_vectorizer (loop_vinfo, vector_iters_vf, + update_e, uncounted_p); /* Recalculate the dominators after adding the guard edge. */ if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)) diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index ab0ae8b185f9..c8662e4b2981 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -2673,9 +2673,11 @@ start_over: niters_th += LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo); } - /* Niters for at least one iteration of vectorized loop. */ + /* Niters for one vector iteration, excluding a header-only iteration. */ if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo)) niters_th += LOOP_VINFO_VECT_FACTOR (loop_vinfo); + if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)) + niters_th += 1; /* One additional iteration because of peeling for gap. */ if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)) niters_th += 1;
