On Wed, 12 Aug 2026, Tamar Christina wrote:
> This adds support for matching FMS where the reassoc has left the top level
> operation as a PLUS_MINUS.
>
> i.e. matching
>
> c.real + (a.imag * b.imag - a.real * b.real)
> c.imag - (a.real * b.imag + a.imag * b.real)
>
> which happens when the accumulator is not the same as the destination.
> i.e.
>
> d[i] c[i] - (a[i] * b[i])
>
> vs
>
> c[i] -= a[i] * b[i]
>
> Bootstrapped Regtested on aarch64-none-linux-gnu,
> arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
> -m32, -m64 and no issues.
>
> Any comments?
LGTM
> Thanks,
> Tamar
>
> gcc/ChangeLog:
>
> * tree-vect-slp-patterns.cc (complex_fms_pattern::matches): Check
> another form of FMS.
>
> gcc/testsuite/ChangeLog:
>
> * gfortran.target/aarch64/pr122408_1.f90: update scans to check for FMS.
> * gcc.dg/vect/complex/complex-mul2.c: New test.
> * gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c: New
> test.
> * gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c: New
> test.
> * gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c: New
> test.
>
> ---
> diff --git a/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
> b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
> new file mode 100644
> index
> 0000000000000000000000000000000000000000..6a45ebdbec8c91dfae98f40f910c7a5d3f2ffbf5
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
> @@ -0,0 +1,67 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */
> +/* { dg-require-effective-target vect_complex_add_double } */
> +/* { dg-add-options arm_v8_3a_complex_neon } */
> +
> +extern void abort (void);
> +
> +#define N 16
> +
> +#define DEF(NAME, TYPE, EXPR) \
> + __attribute__((noipa)) void \
> + NAME (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\
> + _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\
> + int n) \
> + { \
> + for (int i = 0; i < n; i++) \
> + d[i] = EXPR; \
> + } \
> + \
> + __attribute__((noipa, optimize ("no-tree-vectorize"))) void \
> + NAME##_ref (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\
> + _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\
> + int n) \
> + { \
> + for (int i = 0; i < n; i++) \
> + d[i] = EXPR; \
> + }
> +
> +DEF (fms_f, float, c[i] - a[i] * b[i])
> +DEF (fms_d, double, c[i] - a[i] * b[i])
> +DEF (fmsconj_f, float, c[i] - a[i] * ~b[i])
> +DEF (mul_f, float, a[i] * b[i])
> +
> +#define CHECK(NAME, TYPE) \
> + do { \
> + _Complex TYPE a[N], b[N], c[N], d[N], ref[N]; \
> + for (int i = 0; i < N; i++) \
> + { \
> + __real__ a[i] = i + 1; \
> + __imag__ a[i] = 2 * i + 3; \
> + __real__ b[i] = 3 * i - 1; \
> + __imag__ b[i] = i + 5; \
> + __real__ c[i] = 100 + i; \
> + __imag__ c[i] = 200 - i; \
> + } \
> + NAME (d, c, a, b, N); \
> + NAME##_ref (ref, c, a, b, N); \
> + _Pragma("novect") \
> + for (int i = 0; i < N; i++) \
> + if (__real__ d[i] != __real__ ref[i] \
> + || __imag__ d[i] != __imag__ ref[i]) \
> + abort (); \
> + } while (0)
> +
> +int
> +main (void)
> +{
> + CHECK (fms_f, float);
> + CHECK (fms_d, double);
> + CHECK (fmsconj_f, float);
> + CHECK (mul_f, float);
> + return 0;
> +}
> +
> +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\("
> "vect" } } */
> +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS_CONJ"
> "vect" } } */
> +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_MUL \\("
> "vect" } } */
> diff --git
> a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
>
> b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
> new file mode 100644
> index
> 0000000000000000000000000000000000000000..3062878edf6e7dea5b0b51be132d5506b274cb14
> --- /dev/null
> +++
> b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
> @@ -0,0 +1,27 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */
> +/* { dg-require-effective-target vect_complex_add_float } */
> +/* { dg-add-options arm_v8_3a_complex_neon } */
> +
> +void
> +manual_invalid_fms (float *__restrict d, float *__restrict c,
> + float *__restrict a, float *__restrict b)
> +{
> + for (int r = 0; r < 100; r += 2)
> + {
> + int i = r + 1;
> + float cr = c[r];
> + float ci = c[i];
> + float ar = a[r];
> + float ai = a[i];
> + float br = b[r];
> + float bi = b[i];
> + d[r] = cr + ar * br - ai * bi;
> + d[i] = ci - (ar * bi + ai * br);
> + }
> +}
> +
> +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS pattern" "vect" } } */
> +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
> +/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
> +/* { dg-final { scan-tree-dump-not "add new stmt: \[^\n\r]*COMPLEX_FMS"
> "vect" } } */
> diff --git
> a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
> b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
> new file mode 100644
> index
> 0000000000000000000000000000000000000000..3253b3b2d344f450daa96a38f0134328cad6ea92
> --- /dev/null
> +++
> b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros
> -fdump-tree-vect-details" } */
> +/* { dg-require-effective-target vect_complex_add_double } */
> +/* { dg-add-options arm_v8_3a_complex_neon } */
> +
> +void
> +fms_out (_Complex double *__restrict d, _Complex double *__restrict c,
> + _Complex double *__restrict a, _Complex double *__restrict b,
> + int n)
> +{
> + for (int i = 0; i < n; ++i)
> + d[i] = c[i] - a[i] * b[i];
> +}
> +
> +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\("
> "vect" } } */
> +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
> +/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
> diff --git
> a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
> b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
> new file mode 100644
> index
> 0000000000000000000000000000000000000000..51e6b0d8762842ab006115c7299305857fb3a362
> --- /dev/null
> +++
> b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros
> -fdump-tree-vect-details" } */
> +/* { dg-require-effective-target vect_complex_add_float } */
> +/* { dg-add-options arm_v8_3a_complex_neon } */
> +
> +void
> +fms_out (_Complex float *__restrict d, _Complex float *__restrict c,
> + _Complex float *__restrict a, _Complex float *__restrict b,
> + int n)
> +{
> + for (int i = 0; i < n; ++i)
> + d[i] = c[i] - a[i] * b[i];
> +}
> +
> +/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\("
> "vect" { xfail arm*-*-* } } } */
> +/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
> +/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
> diff --git a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
> b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
> index
> b06950d1baae7d8aca2e1c96c8e6e942781b71bd..d62330d4a7760235bf25179b072d986bbe87c653
> 100644
> --- a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
> +++ b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
> @@ -57,10 +57,10 @@ subroutine c_sub_a_conjb(n, a, c, b) ! C -= A * conj(B)
> end do
> end subroutine c_sub_a_conjb
>
> -! The two plain multiplies form .COMPLEX_MUL (#0 + #90) and the two conjugate
> -! multiplies form .COMPLEX_MUL_CONJ (#0 + #270). PR122408 is about detecting
> -! the conjugate form, so the #270 and #180 counts are the ones that guard it.
> -! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
> v[0-9]+.2d, #0} 4 } }
> +! The accumulations form .COMPLEX_FMA (#0 + #90) and .COMPLEX_FMS (#180 +
> +! #270), with the conjugate forms swapping the #90 and #270 rotations.
> PR122408
> +! is about detecting the conjugate form, so the #90 and #270 counts guard it.
> +! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
> v[0-9]+.2d, #0} 2 } }
> ! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
> v[0-9]+.2d, #90} 2 } }
> ! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
> v[0-9]+.2d, #270} 2 } }
> -! { dg-final { scan-assembler-not {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
> v[0-9]+.2d, #180} } }
> +! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
> v[0-9]+.2d, #180} 2 } }
> diff --git a/gcc/tree-vect-slp-patterns.cc b/gcc/tree-vect-slp-patterns.cc
> index
> f77993100ad19ce5502900022a68b39201d24a54..aae000ee5292ccb1521928e43f514a14386a7827
> 100644
> --- a/gcc/tree-vect-slp-patterns.cc
> +++ b/gcc/tree-vect-slp-patterns.cc
> @@ -1305,7 +1305,94 @@ complex_fms_pattern::matches (complex_operation_t op,
> slp_compat_nodes_map_t *compat_cache,
> slp_tree * ref_node, vec<slp_tree> *ops)
> {
> - internal_fn ifn = IFN_LAST;
> + /* It's only valid to form FMSs with -ffp-contract=fast. */
> + if (!SLP_TREE_VECTYPE (*ref_node)
> + || (flag_fp_contract_mode != FP_CONTRACT_FAST
> + && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node))))
> + return IFN_LAST;
> +
> + /* Match c - a * b when SLP has built the result as:
> +
> + c.real + (a.imag * b.imag - a.real * b.real)
> + c.imag - (a.real * b.imag + a.imag * b.real)
> +
> + This represents the same operation as the existing FMS matcher below,
> + but with the accumulator outside the complex product node. */
> + if (op == PLUS_MINUS)
> + {
> + auto plus_ops = SLP_TREE_CHILDREN ((*ops)[0]);
> + auto minus_ops = SLP_TREE_CHILDREN ((*ops)[1]);
> + if (plus_ops.length () != 2 || minus_ops.length () != 2)
> + return IFN_LAST;
> +
> + slp_tree acc = minus_ops[0];
> + slp_tree prod = minus_ops[1];
> + if (!((plus_ops[0] == acc && plus_ops[1] == prod)
> + || (plus_ops[1] == acc && plus_ops[0] == prod)))
> + return IFN_LAST;
> + if (linear_loads_p (perm_cache, acc) != PERM_EVENODD)
> + return IFN_LAST;
> +
> + auto_vec<slp_tree> prod_ops;
> + if (vect_detect_pair_op (prod, true, &prod_ops) != MINUS_PLUS)
> + return IFN_LAST;
> + if (prod_ops.length () != 2)
> + return IFN_LAST;
> +
> + auto prod_left = SLP_TREE_CHILDREN (prod_ops[0]);
> + auto prod_right = SLP_TREE_CHILDREN (prod_ops[1]);
> + if (prod_left.length () != 2
> + || prod_right.length () != 2
> + || !vect_match_expression_p (prod_left[0], MULT_EXPR)
> + || !vect_match_expression_p (prod_left[1], MULT_EXPR)
> + || !vect_match_expression_p (prod_right[0], MULT_EXPR)
> + || !vect_match_expression_p (prod_right[1], MULT_EXPR))
> + return IFN_LAST;
> +
> + auto_vec<slp_tree> left_op, right_op;
> + left_op.safe_splice (SLP_TREE_CHILDREN (prod_left[0]));
> + right_op.safe_splice (SLP_TREE_CHILDREN (prod_left[1]));
> +
> + enum _conj_status status;
> + auto_vec<slp_tree> res_ops;
> + if (!vect_validate_multiplication_commutative (perm_cache,
> compat_cache,
> + right_op, left_op, true,
> + res_ops, &status))
> + return IFN_LAST;
> +
> + internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS
> + : IFN_COMPLEX_FMS_CONJ;
> + if (!vect_pattern_validate_optab (ifn, *ref_node))
> + return IFN_LAST;
> +
> + ops->truncate (0);
> + ops->create (4);
> +
> + complex_perm_kinds_t kind = linear_loads_p (perm_cache, res_ops[0]);
> + if (kind == PERM_EVENODD || kind == PERM_TOP)
> + {
> + ops->quick_push (acc);
> + ops->quick_push (res_ops[0]);
> + ops->quick_push (res_ops[1]);
> + ops->quick_push (res_ops[3]);
> + }
> + else if (kind == PERM_EVENEVEN && status != CONJ_SND)
> + {
> + ops->quick_push (acc);
> + ops->quick_push (res_ops[1]);
> + ops->quick_push (res_ops[0]);
> + ops->quick_push (res_ops[2]);
> + }
> + else
> + {
> + ops->quick_push (acc);
> + ops->quick_push (res_ops[1]);
> + ops->quick_push (res_ops[0]);
> + ops->quick_push (res_ops[3]);
> + }
> +
> + return ifn;
> + }
>
> /* We need to ignore the two_operands nodes that may also match,
> for that we can check if they have any scalar statements and also
> @@ -1318,11 +1405,6 @@ complex_fms_pattern::matches (complex_operation_t op,
> if (!vect_match_expression_p (root, MINUS_EXPR))
> return IFN_LAST;
>
> - /* It's only valid to form FMSs with -ffp-contract=fast. */
> - if (flag_fp_contract_mode != FP_CONTRACT_FAST
> - && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node)))
> - return IFN_LAST;
> -
> /* TODO: Support invariants here, with the new layout CADD now
> can match before we get a chance to try CFMS. */
> auto nodes = SLP_TREE_CHILDREN (root);
> @@ -1352,11 +1434,8 @@ complex_fms_pattern::matches (complex_operation_t op,
> res_ops, &status))
> return IFN_LAST;
>
> - if (status == CONJ_NONE)
> - ifn = IFN_COMPLEX_FMS;
> - else
> - ifn = IFN_COMPLEX_FMS_CONJ;
> -
> + internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS
> + : IFN_COMPLEX_FMS_CONJ;
> if (!vect_pattern_validate_optab (ifn, *ref_node))
> return IFN_LAST;
>
>
>
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Jochen Jaser, Andrew McDonald, Abhinav Puri; (HRB 36809, AG Nuernberg)
