https://gcc.gnu.org/g:ea6244c6f8db7663aca4e684c3105325d8282ede
commit r17-2282-gea6244c6f8db7663aca4e684c3105325d8282ede Author: Philipp Tomsich <[email protected]> Date: Thu Jul 9 18:08:08 2026 +0200 tree-ssa-math-opts: separate FMA-deferring trigger from reassoc reorder param_avoid_fma_max_bits gates two independent transforms: the widening_mul FMA-deferring in tree-ssa-math-opts.cc, which leaves a loop-carried multiply-add as fmul + fadd, and the reassoc loop-carried-FMA reorder added in r14-5779-g746344dd538 (PR tree-optimization/110279), which parallelises 3+ operand chains. Both fire when TYPE_SIZE (elt) <= avoid-fma-max-bits, so for a given type they switch on at the same threshold and a target cannot keep one while dropping the other. This hurts the AArch64 AVOID_CROSS_LOOP_FMA cores (the Ampere-1 family): a 2-operand reduction such as an sgemm inner K-loop is left as fmul + fadd, slower than fmadd on their dispatch-bound pipeline, yet setting avoid-fma-max-bits to 0 to avoid it also disables the reorder. Add a new --param=widening-mul-defer-fma (default 1) that gates only the widening_mul deferring; avoid-fma-max-bits keeps gating the reorder alone. The AVOID_CROSS_LOOP_FMA callback sets the param to 0. No other target and no default behaviour changes. gcc/ChangeLog: * doc/params.texi (widening-mul-defer-fma): Document. * params.opt (-param=widening-mul-defer-fma=): New param. * tree-ssa-math-opts.cc (math_opts_dom_walker::after_dom_children): Gate fma_deferring_state's enable predicate on param_widening_mul_defer_fma on top of param_avoid_fma_max_bits. * config/aarch64/aarch64-tuning-flags.def (AVOID_CROSS_LOOP_FMA): Update the comment for the widening-mul-defer-fma effect. * config/aarch64/aarch64.cc (aarch64_override_options_internal): Inside the AARCH64_EXTRA_TUNE_AVOID_CROSS_LOOP_FMA block, set param_widening_mul_defer_fma to 0. gcc/testsuite/ChangeLog: * gcc.target/aarch64/widening-mul-defer-fma-1.c: New test. * gcc.target/aarch64/widening-mul-defer-fma-2.c: New test. * gcc.target/aarch64/widening-mul-defer-fma-3.c: New test. Diff: --- gcc/config/aarch64/aarch64-tuning-flags.def | 4 ++++ gcc/config/aarch64/aarch64.cc | 12 +++++++++--- gcc/doc/params.texi | 6 ++++++ gcc/params.opt | 6 ++++++ .../gcc.target/aarch64/widening-mul-defer-fma-1.c | 19 +++++++++++++++++++ .../gcc.target/aarch64/widening-mul-defer-fma-2.c | 21 +++++++++++++++++++++ .../gcc.target/aarch64/widening-mul-defer-fma-3.c | 21 +++++++++++++++++++++ gcc/tree-ssa-math-opts.cc | 3 ++- 8 files changed, 88 insertions(+), 4 deletions(-) diff --git a/gcc/config/aarch64/aarch64-tuning-flags.def b/gcc/config/aarch64/aarch64-tuning-flags.def index 058dadecccaa..32bc1c4f3eec 100644 --- a/gcc/config/aarch64/aarch64-tuning-flags.def +++ b/gcc/config/aarch64/aarch64-tuning-flags.def @@ -40,6 +40,10 @@ AARCH64_EXTRA_TUNING_OPTION ("cse_sve_vl_constants", CSE_SVE_VL_CONSTANTS) AARCH64_EXTRA_TUNING_OPTION ("matched_vector_throughput", MATCHED_VECTOR_THROUGHPUT) +/* For cores whose pipeline disfavours loop-carried serial FMAs: set + avoid-fma-max-bits to its maximum (512, i.e. all FMA widths) to enable + the reassoc reorder for 3+ operand chains, and set widening-mul-defer-fma + to 0 to suppress the widening-mul pass's FMA deferring. */ AARCH64_EXTRA_TUNING_OPTION ("avoid_cross_loop_fma", AVOID_CROSS_LOOP_FMA) AARCH64_EXTRA_TUNING_OPTION ("fully_pipelined_fma", FULLY_PIPELINED_FMA) diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index 5c018d929861..f76db91d4238 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -20095,11 +20095,17 @@ aarch64_override_options_internal (struct gcc_options *opts, && opts->x_optimize >= aarch64_tune_params.prefetch->default_opt_level) opts->x_flag_prefetch_loop_arrays = 1; - /* Avoid loop-dependant FMA chains. */ + /* Avoid loop-dependant FMA chains. The reassoc-side reorder helper + keeps using --param=avoid-fma-max-bits; the widening-mul-side + deferring is gated separately by --param=widening-mul-defer-fma, so we + suppress only the deferring on these cores while leaving the reassoc + reorder active. */ if (aarch64_tune_params.extra_tuning_flags & AARCH64_EXTRA_TUNE_AVOID_CROSS_LOOP_FMA) - SET_OPTION_IF_UNSET (opts, opts_set, param_avoid_fma_max_bits, - 512); + { + SET_OPTION_IF_UNSET (opts, opts_set, param_avoid_fma_max_bits, 512); + SET_OPTION_IF_UNSET (opts, opts_set, param_widening_mul_defer_fma, 0); + } /* Consider fully pipelined FMA in reassociation. */ if (aarch64_tune_params.extra_tuning_flags diff --git a/gcc/doc/params.texi b/gcc/doc/params.texi index 28cfe73d9d73..26b34a59aa98 100644 --- a/gcc/doc/params.texi +++ b/gcc/doc/params.texi @@ -1704,6 +1704,12 @@ reassociation considers the benefit of parallelizing FMA's multiplication part and addition part, assuming FMUL and FMA use the same units that can also do FADD. +@paindex widening-mul-defer-fma +@item widening-mul-defer-fma +When nonzero, the widening-multiply pass defers forming an FMA whose result +feeds a loop-header PHI, leaving a separate multiply and add. Set to zero to +contract such loop-carried reductions into an FMA instead. + @paindex sms-loop-average-count-threshold @item sms-loop-average-count-threshold A threshold on the average loop count considered by the swing modulo scheduler. diff --git a/gcc/params.opt b/gcc/params.opt index 90f9943c8cb4..044c4a10bc44 100644 --- a/gcc/params.opt +++ b/gcc/params.opt @@ -1323,4 +1323,10 @@ Maximum number of outgoing edges in a switch before VRP does not process it. Common Joined UInteger Var(param_vrp_vector_threshold) Init(250) Optimization Param Maximum number of basic blocks for VRP to use a basic cache vector. +-param=widening-mul-defer-fma= +Common Joined UInteger Var(param_widening_mul_defer_fma) Init(1) IntegerRange(0, 1) Param Optimization +When nonzero, the widening-multiply pass defers forming an FMA whose result +feeds a loop-header PHI, leaving a separate multiply and add. Set to zero to +contract such loop-carried reductions into an FMA instead. + ; This comment is to ensure we retain the blank line above. diff --git a/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-1.c b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-1.c new file mode 100644 index 000000000000..dbee0ecb62b6 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-1.c @@ -0,0 +1,19 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ffast-math -fno-tree-vectorize -mcpu=ampere1" } */ + +/* The ampere1 family sets AARCH64_EXTRA_TUNE_AVOID_CROSS_LOOP_FMA, which + sets widening-mul-defer-fma to 0 so that a 2-operand loop-carried + reduction is contracted to fmadd rather than deferred to fmul + fadd. + (avoid-fma-max-bits stays 512 to keep the reassoc reorder enabled.) */ + +double +dot (const double *a, const double *b, int n) +{ + double s = 0.0; + for (int i = 0; i < n; i++) + s += a[i] * b[i]; + return s; +} + +/* { dg-final { scan-assembler {\tfmadd\t} } } */ +/* { dg-final { scan-assembler-not {\tfmul\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-2.c b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-2.c new file mode 100644 index 000000000000..a73d1ac1e9d3 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-2.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ffast-math -fno-tree-vectorize --param=avoid-fma-max-bits=512" } */ + +/* With FMA deferring active (avoid-fma-max-bits > 0) and the default + widening-mul-defer-fma=1, the widening_mul pass refuses to form an FMA + whose result feeds the loop-header phi: the reduction stays as a + separate fmul + fadd. This is the behaviour the new param decouples + from the reassoc reorder (also gated by avoid-fma-max-bits). */ + +double +dot (const double *a, const double *b, int n) +{ + double s = 0.0; + for (int i = 0; i < n; i++) + s += a[i] * b[i]; + return s; +} + +/* { dg-final { scan-assembler {\tfmul\t} } } */ +/* { dg-final { scan-assembler {\tfadd\t} } } */ +/* { dg-final { scan-assembler-not {\tfmadd\t} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-3.c b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-3.c new file mode 100644 index 000000000000..6463ca617734 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/widening-mul-defer-fma-3.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ffast-math -fno-tree-vectorize --param=avoid-fma-max-bits=512 --param=widening-mul-defer-fma=0" } */ + +/* Companion to widening-mul-defer-fma-2.c: the same reduction with FMA + avoidance still requested for the reassoc reorder (avoid-fma-max-bits=512) + but --param=widening-mul-defer-fma=0 now suppresses the widening_mul + deferring, so the loop-carried multiply-add is contracted to a single + fmadd. This proves the new param decouples the deferring from + avoid-fma-max-bits. */ + +double +dot (const double *a, const double *b, int n) +{ + double s = 0.0; + for (int i = 0; i < n; i++) + s += a[i] * b[i]; + return s; +} + +/* { dg-final { scan-assembler {\tfmadd\t} } } */ +/* { dg-final { scan-assembler-not {\tfmul\t} } } */ diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc index f0ede668d95e..c4a1d7bcf0bd 100644 --- a/gcc/tree-ssa-math-opts.cc +++ b/gcc/tree-ssa-math-opts.cc @@ -6607,7 +6607,8 @@ math_opts_dom_walker::after_dom_children (basic_block bb) { gimple_stmt_iterator gsi; - fma_deferring_state fma_state (param_avoid_fma_max_bits > 0); + fma_deferring_state fma_state (param_avoid_fma_max_bits > 0 + && param_widening_mul_defer_fma); for (gphi_iterator psi_next, psi = gsi_start_phis (bb); !gsi_end_p (psi); psi = psi_next)
