https://gcc.gnu.org/g:2e239c8d557d154c745f1d6d2b66d95e05a2f50d
commit r17-5530-g2e239c8d557d154c745f1d6d2b66d95e05a2f50d Author: Tamar Christina <[email protected]> Date: Sat Oct 24 11:39:08 2026 +0100 AArch64: Refactor vector statement costing to use fractional_cost The AArch64 cost model has a helper called aarch64_ifn_vect_stmt_p which accounts for vector statements that expand to more than one instruction. It currently returns the number of extra instructions, leaving its callers to turn that into a cost. This is rather restrictive when we want to adjust the cost of a statement directly. The rest of the cost model already uses fractional_cost, so this patch changes the helper to take the current cost and return the adjusted cost using that same type. It is renamed to aarch64_vect_stmt_cost to reflect what it now does. For example, a statement that needs two extra operations used to return 2. It now returns three times the supplied cost. The general-operation counter passes a unit cost, while the statement costing path passes the cost it has already calculated. Statements that need no adjustment simply return that supplied cost unchanged. There is also a counting issue in the old statement costing path. It multiplies the cost of the extra instructions by count, but record_stmt_cost later multiplies the whole statement cost by count again. For a unit base cost, two extra operations and four copies, this gives (1 + 4 * 2) * 4 = 36 rather than the expected (1 + 2) * 4 = 12 Returning the adjusted cost of one statement lets record_stmt_cost apply the copy count once. The general-operation counter still scales its extra operations by count and rounds up the result after that multiplication. This also gives the dotproduct accumulator costing patch from Konstantino a place to extend rather than inventing a new helper. gcc/ChangeLog: * config/aarch64/aarch64.cc (aarch64_ifn_vect_stmt_p): Rename to... (aarch64_vect_stmt_cost): ...this. Accept a fractional_cost and return the adjusted statement cost instead of an additional instruction count. (aarch64_vector_costs::count_ops): Pass a unit cost and scale and round the extra operation count after applying count. (aarch64_vector_costs::add_stmt_cost): Use the adjusted statement cost and leave count scaling to record_stmt_cost. Diff: --- gcc/config/aarch64/aarch64.cc | 52 ++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index 5b1c78462a37..9290677655aa 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -18488,20 +18488,20 @@ aarch64_sve_adjust_stmt_cost (class vec_info *vinfo, vect_cost_for_stmt kind, return stmt_cost; } -/* Return > 0 if STMT_INFO is the instruction requires additional costing. The - instruction is evaluated as being vectorized as VECTYPE and the return value - should be the additional number of instructions that need to be costed. */ -static unsigned int -aarch64_ifn_vect_stmt_p (stmt_vec_info stmt_info, tree vectype) +/* Return the cost of implementing STMT_INFO using VECTYPE, given the + default cost STMT_COST. */ +static fractional_cost +aarch64_vect_stmt_cost (stmt_vec_info stmt_info, tree vectype, + fractional_cost stmt_cost) { stmt_info = vect_stmt_to_vectorize (stmt_info); gcall *call = dyn_cast<gcall *> (STMT_VINFO_STMT (stmt_info)); if (!vectype) - return 0; + return stmt_cost; gassign *assign = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_info)); if (!call && !assign) - return 0; + return stmt_cost; auto vec_flags = aarch64_classify_vector_mode (TYPE_MODE (vectype)); bool advsimd_p = vec_flags & VEC_ADVSIMD; @@ -18510,10 +18510,9 @@ aarch64_ifn_vect_stmt_p (stmt_vec_info stmt_info, tree vectype) { switch (gimple_assign_rhs_code (assign)) { - /* SAD for Adv. SIMD is emulated using two instuctions per 64-bit - quantities. So 128-bit ADB requires 4 INSN. Account for that. */ case SAD_EXPR: - return advsimd_p && is_128bit_p ? 2 : 0; + return stmt_cost * (advsimd_p && is_128bit_p ? 3 : 1); + case WIDEN_SUM_EXPR: { tree rhs = gimple_assign_rhs1 (assign); @@ -18521,7 +18520,7 @@ aarch64_ifn_vect_stmt_p (stmt_vec_info stmt_info, tree vectype) || TARGET_DOTPROD || !vect_is_reduction (stmt_info) || TREE_CODE (rhs) != SSA_NAME) - return 0; + return stmt_cost; gimple *def_stmt = SSA_NAME_DEF_STMT (rhs); gassign *def_assign = dyn_cast<gassign *> (def_stmt); @@ -18532,21 +18531,21 @@ aarch64_ifn_vect_stmt_p (stmt_vec_info stmt_info, tree vectype) gcall *def = dyn_cast<gcall *> (def_stmt); if (!def || gimple_call_combined_fn (def) != CFN_ABD) - return 0; + return stmt_cost; for (unsigned int i = 0; i < 2; ++i) { tree arg = gimple_call_arg (def, i); if (TREE_CODE (arg) != SSA_NAME || !gimple_assign_load_p (SSA_NAME_DEF_STMT (arg))) - return 0; + return stmt_cost; } - return 2; + return stmt_cost * 3; } default: break; } - return 0; + return stmt_cost; } switch (gimple_call_combined_fn (call)) @@ -18554,7 +18553,7 @@ aarch64_ifn_vect_stmt_p (stmt_vec_info stmt_info, tree vectype) default: break; } - return 0; + return stmt_cost; } /* STMT_COST is the cost calculated for STMT_INFO, which has cost kind KIND @@ -18682,12 +18681,11 @@ aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind, return; } - unsigned int n_insn = 0; - if (stmt_info - && kind == vector_stmt - && (n_insn = aarch64_ifn_vect_stmt_p (stmt_info, - STMT_VINFO_VECTYPE (stmt_info)))) - ops->general_ops += n_insn * count; + if (stmt_info && kind == vector_stmt) + ops->general_ops + += ((aarch64_vect_stmt_cost (stmt_info, + STMT_VINFO_VECTYPE (stmt_info), 1) - 1) + * count).ceil (); /* Detect the case where we are using an emulated gather/scatter. When a target does not support gathers and scatters directly the vectorizer @@ -19270,14 +19268,8 @@ aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, to the base cost calculated above. */ stmt_cost = aarch64_adjust_stmt_cost (m_vinfo, kind, stmt_info, node, vectype, m_vec_flags, stmt_cost); - unsigned int n_insn = 0; - if (vectype - && kind == vector_stmt - && (n_insn = aarch64_ifn_vect_stmt_p (stmt_info, vectype))) - { - const simd_vec_cost *simd_costs = aarch64_simd_vec_costs (vectype); - stmt_cost += count * n_insn * simd_costs->int_stmt_cost; - } + if (vectype && kind == vector_stmt) + stmt_cost = aarch64_vect_stmt_cost (stmt_info, vectype, stmt_cost); /* If we're applying the SVE vs. Advanced SIMD unrolling heuristic, estimate the number of statements in the unrolled Advanced SIMD
