https://gcc.gnu.org/g:8c46d2fae76b2ddaafec2f14121469f7dd67f745
commit r17-5332-g8c46d2fae76b2ddaafec2f14121469f7dd67f745 Author: Tamar Christina <[email protected]> Date: Mon Oct 19 12:52:32 2026 +0100 vect: don't create unsupported reduction recurrence in ifcvt [PR126551] The example float f (const float *x, int n) { float acc = 0; for (int i = 0; i < n; ++i) { float d = x[i]; if (d < 0) acc += d; acc += d * 3.0f; } return acc; } compiled at -Ofast fails to vectorize with note: vect_is_simple_use: operand acc_17 = PHI <acc_13(8), 0.0(18)>, type of def: unknown missed: Unsupported pattern. missed: not vectorized: unsupported use in stmt. due to note: Analyze phi: acc_17 = PHI <acc_13(8), 0.0(18)> missed: reduction used in loop. missed: Unknown def-use cycle pattern. This is happening because before if-convert we have the following reduction: acc_1 = PHI <acc_0, acc_2> if (cond) acc_2 = acc_1 + value; and so the reduction value is updated only on some paths. if-cvt changes this into acc_3 = cond ? acc_2 : acc_1; but since acc_2 itself is acc_1 + value this is actually acc_3 = cond ? acc_1 + value : acc_1; This is semantically sound but hides the recurrence from the vectorizer. In principle if-cvt creates acc_1 = PHI <0, acc_3> acc_2 = acc_1 + value acc_3 = cond ? acc_2 : acc_1 so the loop carried value acc_3 is inside the COND_EXPR and acc_1 is used twice inside the same reduction. The update chain is PHI -> PLUS -> COND_EXPR -> PHI \--------------^ which is a problem since reduction analysis needs a single statement that updates the loop carried accumulator. Note that if the user supplied this form manually, if-cvt rewrites it to one of two forms that the vectorizer can handle. It either rewrites it to COND_ADD and this removing the PLUS + COND_EXPR or it floats out the cond_expr. This patch does the latter, as picking COND_ADD here is premature, it forces selection of a masked target even though the vectorizer these days can convert the cond expr to a masked operation on its own based on costing. Essentially we change cond ? base + addend : base or cond ? base : base + addend into base + (cond ? addend : 0) or base + (cond ? 0 : addend) -0.0 is used for FP when signed zeros are being honored. this turns the recurrence into a form the vectorizer can handle. gcc/ChangeLog: PR tree-optimization/126551 * tree-if-conv.cc (is_cond_scalar_reduction): Recognize intermediate conditional additions in reduction update chains. (convert_scalar_cond_reduction): Keep the addition unconditional for intermediate reduction. (predicate_scalar_phi): Pass the intermediate flag. gcc/testsuite/ChangeLog: PR tree-optimization/126551 * gcc.dg/vect/cond-reduc-ifcvt-1.c: New test. * gcc.dg/vect/cond-reduc-ifcvt-2.c: Likewise. * gcc.dg/vect/cond-reduc-ifcvt-3.c: Likewise. * gcc.dg/vect/cond-reduc-ifcvt-4.c: Likewise. * gcc.dg/vect/cond-reduc-ifcvt-5.c: Likewise. * gcc.dg/vect/cond-reduc-ifcvt-6.c: Likewise. * gcc.dg/vect/vect-switch-ifcvt-2.c: Expect one loop to be vectorized. * gcc.target/aarch64/sve/cond-reduc-ifcvt-trap.c: New test. Diff: --- gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-1.c | 21 +++++++++ gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-2.c | 21 +++++++++ gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-3.c | 21 +++++++++ gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-4.c | 22 ++++++++++ gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-5.c | 24 ++++++++++ gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-6.c | 25 +++++++++++ gcc/testsuite/gcc.dg/vect/vect-switch-ifcvt-2.c | 2 +- .../gcc.target/aarch64/sve/cond-reduc-ifcvt-trap.c | 21 +++++++++ gcc/tree-if-conv.cc | 51 +++++++++++++++++----- 9 files changed, 195 insertions(+), 13 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-1.c b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-1.c new file mode 100644 index 000000000000..d5bf28a7afd9 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-1.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_float } */ +/* { dg-additional-options "-Ofast -fdump-tree-ifcvt-details" } */ + +float +f (const float *x, int n) +{ + float acc = 0; + for (int i = 0; i < n; ++i) + { + float d = x[i]; + if (d < 0) + acc += d; + acc += d * 3.0f; + } + return acc; +} + +/* { dg-final { scan-tree-dump "new phi replacement stmt" "ifcvt" } } */ +/* { dg-final { scan-tree-dump "_ifc_.* = MIN_EXPR" "ifcvt" } } */ +/* { dg-final { scan-tree-dump { = (acc_.* \+ _ifc_|_ifc_.* \+ acc_)} "ifcvt" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-2.c b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-2.c new file mode 100644 index 000000000000..6a945afe08c6 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-2.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_int } */ +/* { dg-additional-options "-O3 -fdump-tree-ifcvt-details" } */ + +int +f (const int *x, int n) +{ + int acc = 0; + for (int i = 0; i < n; ++i) + { + int d = x[i]; + if (d < 0) + acc += d; + acc += d * 3; + } + return acc; +} + +/* { dg-final { scan-tree-dump "new phi replacement stmt" "ifcvt" } } */ +/* { dg-final { scan-tree-dump "_ifc_.* = MIN_EXPR" "ifcvt" } } */ +/* { dg-final { scan-tree-dump { = (acc_.* \+ _ifc_|_ifc_.* \+ acc_)} "ifcvt" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-3.c b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-3.c new file mode 100644 index 000000000000..eda48af037c8 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-3.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_float } */ +/* { dg-additional-options "-O3 -fno-associative-math -fsigned-zeros -fno-trapping-math -fdump-tree-ifcvt-details" } */ + +float +f (const float *x, float init, int n) +{ + float acc = init; + for (int i = 0; i < n; ++i) + { + float d = x[i]; + if (d < 0) + acc += d; + acc += d * 3.0f; + } + return acc; +} + +/* The inactive addend must preserve a negative zero in ACC. */ +/* { dg-final { scan-tree-dump "new phi replacement stmt" "ifcvt" } } */ +/* { dg-final { scan-tree-dump "_ifc_.* = .* \\? .* : -0.0;" "ifcvt" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-4.c b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-4.c new file mode 100644 index 000000000000..7441a3c02b64 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-4.c @@ -0,0 +1,22 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_float } */ +/* { dg-additional-options "-O3 -fno-trapping-math -fdump-tree-ifcvt-details" } */ + +float +f (const float *x, int n) +{ + float acc = 0; + for (int i = 0; i < n; ++i) + { + float d = x[i]; + if (d < 0) + acc += d; + acc = __builtin_fmaxf (acc, d * 3.0f); + } + return acc; +} + +/* The latch update is a call, or a MAX_EXPR if the call is folded. */ +/* { dg-final { scan-tree-dump "new phi replacement stmt" "ifcvt" } } */ +/* { dg-final { scan-tree-dump { = (acc_.* \+ _ifc_|_ifc_.* \+ acc_)} "ifcvt" } } */ +/* { dg-final { scan-tree-dump "((__builtin_)?fmaxf|MAX_EXPR)" "ifcvt" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-5.c b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-5.c new file mode 100644 index 000000000000..5bbf88692823 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-5.c @@ -0,0 +1,24 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_float } */ +/* { dg-additional-options "-O3 -fno-associative-math -frounding-math -fsigned-zeros -fno-trapping-math -fdump-tree-ifcvt-details" } */ + +float +f (const float *x, float init, int n) +{ + float acc = init; + for (int i = 0; i < n; ++i) + { + float d = x[i]; + if (d < 0) + acc += d; + acc += d * 3.0f; + } + return acc; +} + +/* With rounding math, an inactive addition of -0.0 need not preserve ACC. + Disable trapping math so the addition reaches PHI conversion as an + assignment, rather than being predicated using a conditional call. */ +/* { dg-final { scan-tree-dump "new phi replacement stmt" "ifcvt" } } */ +/* { dg-final { scan-tree-dump {acc_[0-9]+ = [^\n]* \? acc_[0-9]+ : acc_[0-9]+;} "ifcvt" } } */ +/* { dg-final { scan-tree-dump-not "Found cond scalar reduction" "ifcvt" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-6.c b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-6.c new file mode 100644 index 000000000000..855699153584 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/cond-reduc-ifcvt-6.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_float } */ +/* { dg-require-effective-target vect_condition } */ +/* { dg-additional-options "-Ofast -fno-tree-reassoc -fdump-tree-ifcvt-details" } */ + +float +f (const float *x, const float *y, const float *z, int n) +{ + float acc = 0; + for (int i = 0; i < n; ++i) + { + float d = x[i]; + if (d < 0) + acc += d; + acc += y[i]; + acc += z[i]; + } + return acc; +} + +/* Keep both unconditional additions, so the conditional PHI does not + directly feed the statement that produces the latch value. */ +/* { dg-final { scan-tree-dump "Found cond scalar reduction" "ifcvt" } } */ +/* { dg-final { scan-tree-dump "_ifc_.* = MIN_EXPR" "ifcvt" } } */ +/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/vect-switch-ifcvt-2.c b/gcc/testsuite/gcc.dg/vect/vect-switch-ifcvt-2.c index f0fcb43e1770..03f241aef375 100644 --- a/gcc/testsuite/gcc.dg/vect/vect-switch-ifcvt-2.c +++ b/gcc/testsuite/gcc.dg/vect/vect-switch-ifcvt-2.c @@ -46,4 +46,4 @@ f2 (char *s) return c; } -/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 0 "vect" } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond-reduc-ifcvt-trap.c b/gcc/testsuite/gcc.target/aarch64/sve/cond-reduc-ifcvt-trap.c new file mode 100644 index 000000000000..ce3c70e649e1 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/cond-reduc-ifcvt-trap.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-options "-O3 -march=armv8.2-a+sve -ftrapping-math -frounding-math -fno-associative-math -fsigned-zeros -fdump-tree-ifcvt-details" } */ + +float +f (const float *x, float init, int n) +{ + float acc = init; + for (int i = 0; i < n; ++i) + { + float d = x[i]; + if (d < 0) + acc += d; + acc += d * 3.0f; + } + return acc; +} + +/* Rounding math prevents replacing the guarded addition with an addition + of -0.0, but SVE can perform the addition only when the condition holds. */ +/* { dg-final { scan-tree-dump {\.COND_ADD \(} "ifcvt" } } */ +/* { dg-final { scan-tree-dump-not {_ifc_.* = .* : -0.0;} "ifcvt" } } */ diff --git a/gcc/tree-if-conv.cc b/gcc/tree-if-conv.cc index bd1430e68451..8c104a93d7a5 100644 --- a/gcc/tree-if-conv.cc +++ b/gcc/tree-if-conv.cc @@ -1815,12 +1815,13 @@ strip_nop_cond_scalar_reduction (bool has_nop, tree op) ARG_0 and ARG_1 are correspondent PHI arguments. REDUC, OP0 and OP1 contain reduction stmt and its operands. + INTERMEDIATE is true if PHI does not directly feed the loop-header PHI. EXTENDED is true if PHI has > 2 arguments. */ static bool is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1, tree *op0, tree *op1, bool extended, bool* has_nop, - gimple **nop_reduc) + gimple **nop_reduc, bool *intermediate) { tree lhs, r_op1, r_op2, r_nop1, r_nop2; gimple *stmt; @@ -1833,7 +1834,7 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1, use_operand_p use_p; edge e; edge_iterator ei; - bool result = *has_nop = false; + bool result = *has_nop = *intermediate = false; if (TREE_CODE (arg_0) != SSA_NAME || TREE_CODE (arg_1) != SSA_NAME) return false; @@ -1854,8 +1855,8 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1, if (gimple_bb (header_phi) != loop->header) return false; - if (PHI_ARG_DEF_FROM_EDGE (header_phi, latch_e) != PHI_RESULT (phi)) - return false; + tree latch_value = PHI_ARG_DEF_FROM_EDGE (header_phi, latch_e); + *intermediate = latch_value != PHI_RESULT (phi); if (gimple_code (stmt) != GIMPLE_ASSIGN || gimple_has_volatile_ops (stmt)) @@ -1919,6 +1920,12 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1, reduction_op = gimple_assign_rhs_code (stmt); } + /* A reduction with a further latch update must keep the PLUS outside + the conditional expression. The no-op conversion case is not needed + for this form. */ + if (*intermediate && (*has_nop || reduction_op != PLUS_EXPR)) + return false; + if (reduction_op != PLUS_EXPR && reduction_op != MINUS_EXPR && reduction_op != MULT_EXPR @@ -1942,6 +1949,21 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1, else if (r_nop1 != PHI_RESULT (header_phi)) return false; + if (*intermediate) + { + tree type = TREE_TYPE (gimple_assign_lhs (stmt)); + if (!SCALAR_FLOAT_TYPE_P (type) + && (!INTEGRAL_TYPE_P (type) || TYPE_OVERFLOW_TRAPS (type))) + return false; + + tree zero = neutral_op_for_reduction (type, PLUS_EXPR, NULL_TREE, + false); + /* The inactive addend must leave the reduction value unchanged. */ + if (SCALAR_FLOAT_TYPE_P (type) + && !fold_real_zero_addition_p (type, r_op1, zero, 0)) + return false; + } + if (*has_nop) { /* Check that R_NOP1 is used in nop_stmt or in PHI only. */ @@ -1989,7 +2011,8 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1, _ifc__1 = _5 != 0 ? 1 : 0; res_2 = res_13 + _ifc__1; Argument SWAP tells that arguments of conditional expression should be - swapped. + swapped. INTERMEDIATE says that this is not the latch value, so the + PLUS must remain unconditional. We can assume that we versioned the loop for vectorization, so can create a COND_OP. Returns rhs of resulting PHI assignment. */ @@ -1997,7 +2020,8 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1, static tree convert_scalar_cond_reduction (gimple *reduc, gimple_stmt_iterator *gsi, tree cond, tree op0, tree op1, bool swap, - bool has_nop, gimple* nop_reduc) + bool has_nop, gimple* nop_reduc, + bool intermediate) { gimple_stmt_iterator stmt_it; gimple *new_assign; @@ -2021,7 +2045,8 @@ convert_scalar_cond_reduction (gimple *reduc, gimple_stmt_iterator *gsi, The COND_OP will have a neutral_op else value. */ internal_fn ifn; ifn = get_conditional_internal_fn (reduction_op); - if (ifn != IFN_LAST + if (!intermediate + && ifn != IFN_LAST && vectorized_internal_fn_supported_p (ifn, TREE_TYPE (lhs)) && !VECTOR_TYPE_P (TREE_TYPE (lhs)) && !swap) @@ -2406,7 +2431,7 @@ predicate_scalar_phi (gphi *phi, gimple_stmt_iterator *gsi) edge e; basic_block bb; unsigned int i; - bool has_nop; + bool has_nop, intermediate; res = gimple_phi_result (phi); if (virtual_operand_p (res)) @@ -2477,12 +2502,12 @@ predicate_scalar_phi (gphi *phi, gimple_stmt_iterator *gsi) if (is_cond_scalar_reduction (phi, &reduc, arg0, arg1, &op0, &op1, false, &has_nop, - &nop_reduc)) + &nop_reduc, &intermediate)) { /* Convert reduction stmt into vectorizable form. */ rhs = convert_scalar_cond_reduction (reduc, gsi, cond, op0, op1, true_bb != gimple_bb (reduc), - has_nop, nop_reduc); + has_nop, nop_reduc, intermediate); redundant_ssa_names.safe_push (std::make_pair (res, rhs)); } else @@ -2572,7 +2597,8 @@ predicate_scalar_phi (gphi *phi, gimple_stmt_iterator *gsi) cond = force_gimple_operand_gsi (gsi, unshare_expr (cond), true, NULL_TREE, true, GSI_SAME_STMT); if (!(is_cond_scalar_reduction (phi, &reduc, arg0 , arg1, - &op0, &op1, true, &has_nop, &nop_reduc))) + &op0, &op1, true, &has_nop, &nop_reduc, + &intermediate))) rhs = fold_build_cond_expr (TREE_TYPE (res), unshare_expr (cond), swap ? arg1 : arg0, swap ? arg0 : arg1); @@ -2580,7 +2606,8 @@ predicate_scalar_phi (gphi *phi, gimple_stmt_iterator *gsi) { /* Convert reduction stmt into vectorizable form. */ rhs = convert_scalar_cond_reduction (reduc, gsi, cond, op0, op1, - swap, has_nop, nop_reduc); + swap, has_nop, nop_reduc, + intermediate); redundant_ssa_names.safe_push (std::make_pair (res, rhs)); } new_stmt = gimple_build_assign (res, rhs);
