https://gcc.gnu.org/g:d9b294889b14069d6f6c5ce985f17bf8bbf34894
commit r17-3228-gd9b294889b14069d6f6c5ce985f17bf8bbf34894 Author: Tamar Christina <[email protected]> Date: Wed Aug 12 14:15:43 2026 +0100 vect: Support another form of COMPLEX_FMS which the testcases point out [PR126589] This adds support for matching FMS where the reassoc has left the top level operation as a PLUS_MINUS. i.e. matching c.real + (a.imag * b.imag - a.real * b.real) c.imag - (a.real * b.imag + a.imag * b.real) which happens when the accumulator is not the same as the destination. i.e. d[i] = c[i] - (a[i] * b[i]) vs c[i] -= a[i] * b[i] gcc/ChangeLog: PR tree-optimization/126589 * tree-vect-slp-patterns.cc (complex_fms_pattern::matches): Check another form of FMS. gcc/testsuite/ChangeLog: PR tree-optimization/126589 * gfortran.target/aarch64/pr122408_1.f90: update scans to check for FMS. * gcc.dg/vect/complex/complex-mul2.c: New test. * gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c: New test. * gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c: New test. * gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c: New test. Diff: --- gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c | 67 ++++++++++++++ .../fast-math-complex-mls-manual-invalid-float.c | 27 ++++++ .../fast-math-complex-mls-outofplace-double.c | 17 ++++ .../fast-math-complex-mls-outofplace-float.c | 17 ++++ .../gfortran.target/aarch64/pr122408_1.f90 | 10 +- gcc/tree-vect-slp-patterns.cc | 101 ++++++++++++++++++--- 6 files changed, 223 insertions(+), 16 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c new file mode 100644 index 000000000000..6a45ebdbec8c --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c @@ -0,0 +1,67 @@ +/* { dg-do compile } */ +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */ +/* { dg-require-effective-target vect_complex_add_double } */ +/* { dg-add-options arm_v8_3a_complex_neon } */ + +extern void abort (void); + +#define N 16 + +#define DEF(NAME, TYPE, EXPR) \ + __attribute__((noipa)) void \ + NAME (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\ + _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\ + int n) \ + { \ + for (int i = 0; i < n; i++) \ + d[i] = EXPR; \ + } \ + \ + __attribute__((noipa, optimize ("no-tree-vectorize"))) void \ + NAME##_ref (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\ + _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\ + int n) \ + { \ + for (int i = 0; i < n; i++) \ + d[i] = EXPR; \ + } + +DEF (fms_f, float, c[i] - a[i] * b[i]) +DEF (fms_d, double, c[i] - a[i] * b[i]) +DEF (fmsconj_f, float, c[i] - a[i] * ~b[i]) +DEF (mul_f, float, a[i] * b[i]) + +#define CHECK(NAME, TYPE) \ + do { \ + _Complex TYPE a[N], b[N], c[N], d[N], ref[N]; \ + for (int i = 0; i < N; i++) \ + { \ + __real__ a[i] = i + 1; \ + __imag__ a[i] = 2 * i + 3; \ + __real__ b[i] = 3 * i - 1; \ + __imag__ b[i] = i + 5; \ + __real__ c[i] = 100 + i; \ + __imag__ c[i] = 200 - i; \ + } \ + NAME (d, c, a, b, N); \ + NAME##_ref (ref, c, a, b, N); \ + _Pragma("novect") \ + for (int i = 0; i < N; i++) \ + if (__real__ d[i] != __real__ ref[i] \ + || __imag__ d[i] != __imag__ ref[i]) \ + abort (); \ + } while (0) + +int +main (void) +{ + CHECK (fms_f, float); + CHECK (fms_d, double); + CHECK (fmsconj_f, float); + CHECK (mul_f, float); + return 0; +} + +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect" } } */ +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS_CONJ" "vect" } } */ +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_MUL \\(" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c new file mode 100644 index 000000000000..3062878edf6e --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c @@ -0,0 +1,27 @@ +/* { dg-do compile } */ +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */ +/* { dg-require-effective-target vect_complex_add_float } */ +/* { dg-add-options arm_v8_3a_complex_neon } */ + +void +manual_invalid_fms (float *__restrict d, float *__restrict c, + float *__restrict a, float *__restrict b) +{ + for (int r = 0; r < 100; r += 2) + { + int i = r + 1; + float cr = c[r]; + float ci = c[i]; + float ar = a[r]; + float ai = a[i]; + float br = b[r]; + float bi = b[i]; + d[r] = cr + ar * br - ai * bi; + d[i] = ci - (ar * bi + ai * br); + } +} + +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS pattern" "vect" } } */ +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */ +/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */ +/* { dg-final { scan-tree-dump-not "add new stmt: \[^\n\r]*COMPLEX_FMS" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c new file mode 100644 index 000000000000..3253b3b2d344 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c @@ -0,0 +1,17 @@ +/* { dg-do compile } */ +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros -fdump-tree-vect-details" } */ +/* { dg-require-effective-target vect_complex_add_double } */ +/* { dg-add-options arm_v8_3a_complex_neon } */ + +void +fms_out (_Complex double *__restrict d, _Complex double *__restrict c, + _Complex double *__restrict a, _Complex double *__restrict b, + int n) +{ + for (int i = 0; i < n; ++i) + d[i] = c[i] - a[i] * b[i]; +} + +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect" } } */ +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */ +/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c new file mode 100644 index 000000000000..51e6b0d87628 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c @@ -0,0 +1,17 @@ +/* { dg-do compile } */ +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros -fdump-tree-vect-details" } */ +/* { dg-require-effective-target vect_complex_add_float } */ +/* { dg-add-options arm_v8_3a_complex_neon } */ + +void +fms_out (_Complex float *__restrict d, _Complex float *__restrict c, + _Complex float *__restrict a, _Complex float *__restrict b, + int n) +{ + for (int i = 0; i < n; ++i) + d[i] = c[i] - a[i] * b[i]; +} + +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect" { xfail arm*-*-* } } } */ +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */ +/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */ diff --git a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90 b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90 index b06950d1baae..d62330d4a776 100644 --- a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90 +++ b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90 @@ -57,10 +57,10 @@ subroutine c_sub_a_conjb(n, a, c, b) ! C -= A * conj(B) end do end subroutine c_sub_a_conjb -! The two plain multiplies form .COMPLEX_MUL (#0 + #90) and the two conjugate -! multiplies form .COMPLEX_MUL_CONJ (#0 + #270). PR122408 is about detecting -! the conjugate form, so the #270 and #180 counts are the ones that guard it. -! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #0} 4 } } +! The accumulations form .COMPLEX_FMA (#0 + #90) and .COMPLEX_FMS (#180 + +! #270), with the conjugate forms swapping the #90 and #270 rotations. PR122408 +! is about detecting the conjugate form, so the #90 and #270 counts guard it. +! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #0} 2 } } ! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #90} 2 } } ! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #270} 2 } } -! { dg-final { scan-assembler-not {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #180} } } +! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #180} 2 } } diff --git a/gcc/tree-vect-slp-patterns.cc b/gcc/tree-vect-slp-patterns.cc index f77993100ad1..aae000ee5292 100644 --- a/gcc/tree-vect-slp-patterns.cc +++ b/gcc/tree-vect-slp-patterns.cc @@ -1305,7 +1305,94 @@ complex_fms_pattern::matches (complex_operation_t op, slp_compat_nodes_map_t *compat_cache, slp_tree * ref_node, vec<slp_tree> *ops) { - internal_fn ifn = IFN_LAST; + /* It's only valid to form FMSs with -ffp-contract=fast. */ + if (!SLP_TREE_VECTYPE (*ref_node) + || (flag_fp_contract_mode != FP_CONTRACT_FAST + && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node)))) + return IFN_LAST; + + /* Match c - a * b when SLP has built the result as: + + c.real + (a.imag * b.imag - a.real * b.real) + c.imag - (a.real * b.imag + a.imag * b.real) + + This represents the same operation as the existing FMS matcher below, + but with the accumulator outside the complex product node. */ + if (op == PLUS_MINUS) + { + auto plus_ops = SLP_TREE_CHILDREN ((*ops)[0]); + auto minus_ops = SLP_TREE_CHILDREN ((*ops)[1]); + if (plus_ops.length () != 2 || minus_ops.length () != 2) + return IFN_LAST; + + slp_tree acc = minus_ops[0]; + slp_tree prod = minus_ops[1]; + if (!((plus_ops[0] == acc && plus_ops[1] == prod) + || (plus_ops[1] == acc && plus_ops[0] == prod))) + return IFN_LAST; + if (linear_loads_p (perm_cache, acc) != PERM_EVENODD) + return IFN_LAST; + + auto_vec<slp_tree> prod_ops; + if (vect_detect_pair_op (prod, true, &prod_ops) != MINUS_PLUS) + return IFN_LAST; + if (prod_ops.length () != 2) + return IFN_LAST; + + auto prod_left = SLP_TREE_CHILDREN (prod_ops[0]); + auto prod_right = SLP_TREE_CHILDREN (prod_ops[1]); + if (prod_left.length () != 2 + || prod_right.length () != 2 + || !vect_match_expression_p (prod_left[0], MULT_EXPR) + || !vect_match_expression_p (prod_left[1], MULT_EXPR) + || !vect_match_expression_p (prod_right[0], MULT_EXPR) + || !vect_match_expression_p (prod_right[1], MULT_EXPR)) + return IFN_LAST; + + auto_vec<slp_tree> left_op, right_op; + left_op.safe_splice (SLP_TREE_CHILDREN (prod_left[0])); + right_op.safe_splice (SLP_TREE_CHILDREN (prod_left[1])); + + enum _conj_status status; + auto_vec<slp_tree> res_ops; + if (!vect_validate_multiplication_commutative (perm_cache, compat_cache, + right_op, left_op, true, + res_ops, &status)) + return IFN_LAST; + + internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS + : IFN_COMPLEX_FMS_CONJ; + if (!vect_pattern_validate_optab (ifn, *ref_node)) + return IFN_LAST; + + ops->truncate (0); + ops->create (4); + + complex_perm_kinds_t kind = linear_loads_p (perm_cache, res_ops[0]); + if (kind == PERM_EVENODD || kind == PERM_TOP) + { + ops->quick_push (acc); + ops->quick_push (res_ops[0]); + ops->quick_push (res_ops[1]); + ops->quick_push (res_ops[3]); + } + else if (kind == PERM_EVENEVEN && status != CONJ_SND) + { + ops->quick_push (acc); + ops->quick_push (res_ops[1]); + ops->quick_push (res_ops[0]); + ops->quick_push (res_ops[2]); + } + else + { + ops->quick_push (acc); + ops->quick_push (res_ops[1]); + ops->quick_push (res_ops[0]); + ops->quick_push (res_ops[3]); + } + + return ifn; + } /* We need to ignore the two_operands nodes that may also match, for that we can check if they have any scalar statements and also @@ -1318,11 +1405,6 @@ complex_fms_pattern::matches (complex_operation_t op, if (!vect_match_expression_p (root, MINUS_EXPR)) return IFN_LAST; - /* It's only valid to form FMSs with -ffp-contract=fast. */ - if (flag_fp_contract_mode != FP_CONTRACT_FAST - && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node))) - return IFN_LAST; - /* TODO: Support invariants here, with the new layout CADD now can match before we get a chance to try CFMS. */ auto nodes = SLP_TREE_CHILDREN (root); @@ -1352,11 +1434,8 @@ complex_fms_pattern::matches (complex_operation_t op, res_ops, &status)) return IFN_LAST; - if (status == CONJ_NONE) - ifn = IFN_COMPLEX_FMS; - else - ifn = IFN_COMPLEX_FMS_CONJ; - + internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS + : IFN_COMPLEX_FMS_CONJ; if (!vect_pattern_validate_optab (ifn, *ref_node)) return IFN_LAST;
