https://gcc.gnu.org/g:1dbbdf48e4d191dbde028c8b6dce7960630736ba
commit r17-5732-g1dbbdf48e4d191dbde028c8b6dce7960630736ba Author: wangzicong <[email protected]> Date: Wed Oct 28 15:09:17 2026 +0800 vect: Avoid double weighting scalar inner-loop costs Hi, vect_compute_single_scalar_iteration_cost records the inner-loop frequency in the statement count. For the entries classified as vect_body, the target cost model applies the frequency again when accumulating scalar costs for outer-loop vectorization. I think commit 12dc8ab has fixed this problem, but commit 7eb6d80 reintroduces it. The new method leaves the body frequency adjustment to the target cost model and computes the inner-loop frequency explicitly when replaying scalar entries for peeling, since prologue and epilogue costs do not receive that adjustment. The test is limited to x86. gcc/ChangeLog: * tree-vect-loop.cc (vect_compute_single_scalar_iteration_cost): Record unweighted scalar statement counts. (vect_scalar_peeling_stmt_count): New function. (vect_estimate_min_profitable_iters): Use it when costing scalar prologue and epilogue iterations. gcc/testsuite/ChangeLog: * gcc.target/i386/vect-outer-scalar-cost-1.c: New test. Signed-off-by: wangzicong <[email protected]> Diff: --- .../gcc.target/i386/vect-outer-scalar-cost-1.c | 20 +++++++++++ gcc/tree-vect-loop.cc | 41 ++++++++++++---------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/gcc/testsuite/gcc.target/i386/vect-outer-scalar-cost-1.c b/gcc/testsuite/gcc.target/i386/vect-outer-scalar-cost-1.c new file mode 100644 index 000000000000..10c29b320b10 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/vect-outer-scalar-cost-1.c @@ -0,0 +1,20 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -fdump-tree-vect-details" } */ + +void +addxor (const unsigned int *__restrict a, unsigned int *__restrict out, + unsigned int m, unsigned int n) +{ + if (n == 0) + return; + for (unsigned int r = 0; r < m; ++r) + { + unsigned int h = r + 1; + for (unsigned int i = 0; i < n; ++i) + h = (h + a[i]) ^ i; + out[r] = h; + } +} + +/* { dg-final { scan-tree-dump { 1 times scalar_load costs [0-9]+ in body} "vect" } } */ +/* { dg-final { scan-tree-dump-not { ([2-9]|[1-9][0-9]+) times scalar_load costs [0-9]+ in body} "vect" } } */ diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index e8cff98997cb..ab0ae8b185f9 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -1378,28 +1378,17 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo) { class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo); - int nbbs = loop->num_nodes, factor; - int innerloop_iters, i; + int nbbs = loop->num_nodes, i; DUMP_VECT_SCOPE ("vect_compute_single_scalar_iteration_cost"); /* Gather costs for statements in the scalar loop. */ - /* FORNOW. */ - innerloop_iters = 1; - if (loop->inner) - innerloop_iters = LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo); - for (i = 0; i < nbbs; i++) { gimple_stmt_iterator si; basic_block bb = bbs[i]; - if (bb->loop_father == loop->inner) - factor = innerloop_iters; - else - factor = 1; - for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { gimple *stmt = gsi_stmt (si); @@ -1431,10 +1420,9 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo) else kind = scalar_stmt; - /* We are using vect_prologue here to avoid scaling twice - by the inner loop factor. */ + /* The target cost model applies the inner-loop frequency. */ record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), - factor, kind, stmt_info, 0, vect_body); + 1, kind, stmt_info, 0, vect_body); } } @@ -4046,6 +4034,19 @@ vect_get_known_peeling_cost (loop_vec_info loop_vinfo, int peel_iters_prologue) return retval; } +/* Return the count for SI in PEEL_ITERS scalar iterations, including the + inner-loop frequency. */ + +static int +vect_scalar_peeling_stmt_count (loop_vec_info loop_vinfo, + const stmt_info_for_cost &si, int peel_iters) +{ + int count = si.count; + if (si.stmt_info && stmt_in_inner_loop_p (loop_vinfo, si.stmt_info)) + count *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo); + return count * peel_iters; +} + /* Function vect_estimate_min_profitable_iters Return the number of iterations required for the vector version of the @@ -4251,8 +4252,9 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo, if (peel_iters_prologue) FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si) { - (void) add_stmt_cost (target_cost_data, - si->count * peel_iters_prologue, si->kind, + int count = vect_scalar_peeling_stmt_count (loop_vinfo, *si, + peel_iters_prologue); + (void) add_stmt_cost (target_cost_data, count, si->kind, si->stmt_info, si->node, si->vectype, si->misalign, vect_prologue); } @@ -4261,8 +4263,9 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo, if (peel_iters_epilogue) FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si) { - (void) add_stmt_cost (target_cost_data, - si->count * peel_iters_epilogue, si->kind, + int count = vect_scalar_peeling_stmt_count (loop_vinfo, *si, + peel_iters_epilogue); + (void) add_stmt_cost (target_cost_data, count, si->kind, si->stmt_info, si->node, si->vectype, si->misalign, vect_epilogue); }
