This adds support for matching FMS where the reassoc has left the top level
operation as a PLUS_MINUS.
i.e. matching
c.real + (a.imag * b.imag - a.real * b.real)
c.imag - (a.real * b.imag + a.imag * b.real)
which happens when the accumulator is not the same as the destination.
i.e.
d[i] c[i] - (a[i] * b[i])
vs
c[i] -= a[i] * b[i]
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any comments?
Thanks,
Tamar
gcc/ChangeLog:
* tree-vect-slp-patterns.cc (complex_fms_pattern::matches): Check
another form of FMS.
gcc/testsuite/ChangeLog:
* gfortran.target/aarch64/pr122408_1.f90: update scans to check for FMS.
* gcc.dg/vect/complex/complex-mul2.c: New test.
* gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c: New
test.
* gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c: New
test.
* gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c: New
test.
---
diff --git a/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
new file mode 100644
index
0000000000000000000000000000000000000000..6a45ebdbec8c91dfae98f40f910c7a5d3f2ffbf5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
@@ -0,0 +1,67 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */
+/* { dg-require-effective-target vect_complex_add_double } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+extern void abort (void);
+
+#define N 16
+
+#define DEF(NAME, TYPE, EXPR) \
+ __attribute__((noipa)) void \
+ NAME (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\
+ _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\
+ int n) \
+ { \
+ for (int i = 0; i < n; i++) \
+ d[i] = EXPR; \
+ } \
+ \
+ __attribute__((noipa, optimize ("no-tree-vectorize"))) void \
+ NAME##_ref (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\
+ _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\
+ int n) \
+ { \
+ for (int i = 0; i < n; i++) \
+ d[i] = EXPR; \
+ }
+
+DEF (fms_f, float, c[i] - a[i] * b[i])
+DEF (fms_d, double, c[i] - a[i] * b[i])
+DEF (fmsconj_f, float, c[i] - a[i] * ~b[i])
+DEF (mul_f, float, a[i] * b[i])
+
+#define CHECK(NAME, TYPE) \
+ do { \
+ _Complex TYPE a[N], b[N], c[N], d[N], ref[N]; \
+ for (int i = 0; i < N; i++) \
+ { \
+ __real__ a[i] = i + 1; \
+ __imag__ a[i] = 2 * i + 3; \
+ __real__ b[i] = 3 * i - 1; \
+ __imag__ b[i] = i + 5; \
+ __real__ c[i] = 100 + i; \
+ __imag__ c[i] = 200 - i; \
+ } \
+ NAME (d, c, a, b, N); \
+ NAME##_ref (ref, c, a, b, N); \
+ _Pragma("novect") \
+ for (int i = 0; i < N; i++) \
+ if (__real__ d[i] != __real__ ref[i] \
+ || __imag__ d[i] != __imag__ ref[i]) \
+ abort (); \
+ } while (0)
+
+int
+main (void)
+{
+ CHECK (fms_f, float);
+ CHECK (fms_d, double);
+ CHECK (fmsconj_f, float);
+ CHECK (mul_f, float);
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect"
} } */
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS_CONJ"
"vect" } } */
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_MUL \\(" "vect"
} } */
diff --git
a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
new file mode 100644
index
0000000000000000000000000000000000000000..3062878edf6e7dea5b0b51be132d5506b274cb14
--- /dev/null
+++
b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */
+/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+void
+manual_invalid_fms (float *__restrict d, float *__restrict c,
+ float *__restrict a, float *__restrict b)
+{
+ for (int r = 0; r < 100; r += 2)
+ {
+ int i = r + 1;
+ float cr = c[r];
+ float ci = c[i];
+ float ar = a[r];
+ float ai = a[i];
+ float br = b[r];
+ float bi = b[i];
+ d[r] = cr + ar * br - ai * bi;
+ d[i] = ci - (ar * bi + ai * br);
+ }
+}
+
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS pattern" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
+/* { dg-final { scan-tree-dump-not "add new stmt: \[^\n\r]*COMPLEX_FMS" "vect"
} } */
diff --git
a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
new file mode 100644
index
0000000000000000000000000000000000000000..3253b3b2d344f450daa96a38f0134328cad6ea92
--- /dev/null
+++
b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros
-fdump-tree-vect-details" } */
+/* { dg-require-effective-target vect_complex_add_double } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+void
+fms_out (_Complex double *__restrict d, _Complex double *__restrict c,
+ _Complex double *__restrict a, _Complex double *__restrict b,
+ int n)
+{
+ for (int i = 0; i < n; ++i)
+ d[i] = c[i] - a[i] * b[i];
+}
+
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect"
} } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
diff --git
a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
new file mode 100644
index
0000000000000000000000000000000000000000..51e6b0d8762842ab006115c7299305857fb3a362
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros
-fdump-tree-vect-details" } */
+/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+void
+fms_out (_Complex float *__restrict d, _Complex float *__restrict c,
+ _Complex float *__restrict a, _Complex float *__restrict b,
+ int n)
+{
+ for (int i = 0; i < n; ++i)
+ d[i] = c[i] - a[i] * b[i];
+}
+
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect"
{ xfail arm*-*-* } } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
diff --git a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
index
b06950d1baae7d8aca2e1c96c8e6e942781b71bd..d62330d4a7760235bf25179b072d986bbe87c653
100644
--- a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
+++ b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
@@ -57,10 +57,10 @@ subroutine c_sub_a_conjb(n, a, c, b) ! C -= A * conj(B)
end do
end subroutine c_sub_a_conjb
-! The two plain multiplies form .COMPLEX_MUL (#0 + #90) and the two conjugate
-! multiplies form .COMPLEX_MUL_CONJ (#0 + #270). PR122408 is about detecting
-! the conjugate form, so the #270 and #180 counts are the ones that guard it.
-! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.2d, #0} 4 } }
+! The accumulations form .COMPLEX_FMA (#0 + #90) and .COMPLEX_FMS (#180 +
+! #270), with the conjugate forms swapping the #90 and #270 rotations.
PR122408
+! is about detecting the conjugate form, so the #90 and #270 counts guard it.
+! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.2d, #0} 2 } }
! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.2d, #90} 2 } }
! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.2d, #270} 2 } }
-! { dg-final { scan-assembler-not {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d,
#180} } }
+! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.2d, #180} 2 } }
diff --git a/gcc/tree-vect-slp-patterns.cc b/gcc/tree-vect-slp-patterns.cc
index
f77993100ad19ce5502900022a68b39201d24a54..aae000ee5292ccb1521928e43f514a14386a7827
100644
--- a/gcc/tree-vect-slp-patterns.cc
+++ b/gcc/tree-vect-slp-patterns.cc
@@ -1305,7 +1305,94 @@ complex_fms_pattern::matches (complex_operation_t op,
slp_compat_nodes_map_t *compat_cache,
slp_tree * ref_node, vec<slp_tree> *ops)
{
- internal_fn ifn = IFN_LAST;
+ /* It's only valid to form FMSs with -ffp-contract=fast. */
+ if (!SLP_TREE_VECTYPE (*ref_node)
+ || (flag_fp_contract_mode != FP_CONTRACT_FAST
+ && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node))))
+ return IFN_LAST;
+
+ /* Match c - a * b when SLP has built the result as:
+
+ c.real + (a.imag * b.imag - a.real * b.real)
+ c.imag - (a.real * b.imag + a.imag * b.real)
+
+ This represents the same operation as the existing FMS matcher below,
+ but with the accumulator outside the complex product node. */
+ if (op == PLUS_MINUS)
+ {
+ auto plus_ops = SLP_TREE_CHILDREN ((*ops)[0]);
+ auto minus_ops = SLP_TREE_CHILDREN ((*ops)[1]);
+ if (plus_ops.length () != 2 || minus_ops.length () != 2)
+ return IFN_LAST;
+
+ slp_tree acc = minus_ops[0];
+ slp_tree prod = minus_ops[1];
+ if (!((plus_ops[0] == acc && plus_ops[1] == prod)
+ || (plus_ops[1] == acc && plus_ops[0] == prod)))
+ return IFN_LAST;
+ if (linear_loads_p (perm_cache, acc) != PERM_EVENODD)
+ return IFN_LAST;
+
+ auto_vec<slp_tree> prod_ops;
+ if (vect_detect_pair_op (prod, true, &prod_ops) != MINUS_PLUS)
+ return IFN_LAST;
+ if (prod_ops.length () != 2)
+ return IFN_LAST;
+
+ auto prod_left = SLP_TREE_CHILDREN (prod_ops[0]);
+ auto prod_right = SLP_TREE_CHILDREN (prod_ops[1]);
+ if (prod_left.length () != 2
+ || prod_right.length () != 2
+ || !vect_match_expression_p (prod_left[0], MULT_EXPR)
+ || !vect_match_expression_p (prod_left[1], MULT_EXPR)
+ || !vect_match_expression_p (prod_right[0], MULT_EXPR)
+ || !vect_match_expression_p (prod_right[1], MULT_EXPR))
+ return IFN_LAST;
+
+ auto_vec<slp_tree> left_op, right_op;
+ left_op.safe_splice (SLP_TREE_CHILDREN (prod_left[0]));
+ right_op.safe_splice (SLP_TREE_CHILDREN (prod_left[1]));
+
+ enum _conj_status status;
+ auto_vec<slp_tree> res_ops;
+ if (!vect_validate_multiplication_commutative (perm_cache, compat_cache,
+ right_op, left_op, true,
+ res_ops, &status))
+ return IFN_LAST;
+
+ internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS
+ : IFN_COMPLEX_FMS_CONJ;
+ if (!vect_pattern_validate_optab (ifn, *ref_node))
+ return IFN_LAST;
+
+ ops->truncate (0);
+ ops->create (4);
+
+ complex_perm_kinds_t kind = linear_loads_p (perm_cache, res_ops[0]);
+ if (kind == PERM_EVENODD || kind == PERM_TOP)
+ {
+ ops->quick_push (acc);
+ ops->quick_push (res_ops[0]);
+ ops->quick_push (res_ops[1]);
+ ops->quick_push (res_ops[3]);
+ }
+ else if (kind == PERM_EVENEVEN && status != CONJ_SND)
+ {
+ ops->quick_push (acc);
+ ops->quick_push (res_ops[1]);
+ ops->quick_push (res_ops[0]);
+ ops->quick_push (res_ops[2]);
+ }
+ else
+ {
+ ops->quick_push (acc);
+ ops->quick_push (res_ops[1]);
+ ops->quick_push (res_ops[0]);
+ ops->quick_push (res_ops[3]);
+ }
+
+ return ifn;
+ }
/* We need to ignore the two_operands nodes that may also match,
for that we can check if they have any scalar statements and also
@@ -1318,11 +1405,6 @@ complex_fms_pattern::matches (complex_operation_t op,
if (!vect_match_expression_p (root, MINUS_EXPR))
return IFN_LAST;
- /* It's only valid to form FMSs with -ffp-contract=fast. */
- if (flag_fp_contract_mode != FP_CONTRACT_FAST
- && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node)))
- return IFN_LAST;
-
/* TODO: Support invariants here, with the new layout CADD now
can match before we get a chance to try CFMS. */
auto nodes = SLP_TREE_CHILDREN (root);
@@ -1352,11 +1434,8 @@ complex_fms_pattern::matches (complex_operation_t op,
res_ops, &status))
return IFN_LAST;
- if (status == CONJ_NONE)
- ifn = IFN_COMPLEX_FMS;
- else
- ifn = IFN_COMPLEX_FMS_CONJ;
-
+ internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS
+ : IFN_COMPLEX_FMS_CONJ;
if (!vect_pattern_validate_optab (ifn, *ref_node))
return IFN_LAST;
--
diff --git a/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
new file mode 100644
index 0000000000000000000000000000000000000000..6a45ebdbec8c91dfae98f40f910c7a5d3f2ffbf5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/complex/complex-mul2.c
@@ -0,0 +1,67 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */
+/* { dg-require-effective-target vect_complex_add_double } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+extern void abort (void);
+
+#define N 16
+
+#define DEF(NAME, TYPE, EXPR) \
+ __attribute__((noipa)) void \
+ NAME (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\
+ _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\
+ int n) \
+ { \
+ for (int i = 0; i < n; i++) \
+ d[i] = EXPR; \
+ } \
+ \
+ __attribute__((noipa, optimize ("no-tree-vectorize"))) void \
+ NAME##_ref (_Complex TYPE *__restrict d, _Complex TYPE *__restrict c,\
+ _Complex TYPE *__restrict a, _Complex TYPE *__restrict b,\
+ int n) \
+ { \
+ for (int i = 0; i < n; i++) \
+ d[i] = EXPR; \
+ }
+
+DEF (fms_f, float, c[i] - a[i] * b[i])
+DEF (fms_d, double, c[i] - a[i] * b[i])
+DEF (fmsconj_f, float, c[i] - a[i] * ~b[i])
+DEF (mul_f, float, a[i] * b[i])
+
+#define CHECK(NAME, TYPE) \
+ do { \
+ _Complex TYPE a[N], b[N], c[N], d[N], ref[N]; \
+ for (int i = 0; i < N; i++) \
+ { \
+ __real__ a[i] = i + 1; \
+ __imag__ a[i] = 2 * i + 3; \
+ __real__ b[i] = 3 * i - 1; \
+ __imag__ b[i] = i + 5; \
+ __real__ c[i] = 100 + i; \
+ __imag__ c[i] = 200 - i; \
+ } \
+ NAME (d, c, a, b, N); \
+ NAME##_ref (ref, c, a, b, N); \
+ _Pragma("novect") \
+ for (int i = 0; i < N; i++) \
+ if (__real__ d[i] != __real__ ref[i] \
+ || __imag__ d[i] != __imag__ ref[i]) \
+ abort (); \
+ } while (0)
+
+int
+main (void)
+{
+ CHECK (fms_f, float);
+ CHECK (fms_d, double);
+ CHECK (fmsconj_f, float);
+ CHECK (mul_f, float);
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect" } } */
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_MUL \\(" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
new file mode 100644
index 0000000000000000000000000000000000000000..3062878edf6e7dea5b0b51be132d5506b274cb14
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-manual-invalid-float.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros" } */
+/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+void
+manual_invalid_fms (float *__restrict d, float *__restrict c,
+ float *__restrict a, float *__restrict b)
+{
+ for (int r = 0; r < 100; r += 2)
+ {
+ int i = r + 1;
+ float cr = c[r];
+ float ci = c[i];
+ float ar = a[r];
+ float ai = a[i];
+ float br = b[r];
+ float bi = b[i];
+ d[r] = cr + ar * br - ai * bi;
+ d[i] = ci - (ar * bi + ai * br);
+ }
+}
+
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS pattern" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
+/* { dg-final { scan-tree-dump-not "add new stmt: \[^\n\r]*COMPLEX_FMS" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
new file mode 100644
index 0000000000000000000000000000000000000000..3253b3b2d344f450daa96a38f0134328cad6ea92
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-double.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros -fdump-tree-vect-details" } */
+/* { dg-require-effective-target vect_complex_add_double } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+void
+fms_out (_Complex double *__restrict d, _Complex double *__restrict c,
+ _Complex double *__restrict a, _Complex double *__restrict b,
+ int n)
+{
+ for (int i = 0; i < n; ++i)
+ d[i] = c[i] - a[i] * b[i];
+}
+
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
new file mode 100644
index 0000000000000000000000000000000000000000..51e6b0d8762842ab006115c7299305857fb3a362
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-outofplace-float.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -fcx-limited-range -fno-signed-zeros -fdump-tree-vect-details" } */
+/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+void
+fms_out (_Complex float *__restrict d, _Complex float *__restrict c,
+ _Complex float *__restrict a, _Complex float *__restrict b,
+ int n)
+{
+ for (int i = 0; i < n; ++i)
+ d[i] = c[i] - a[i] * b[i];
+}
+
+/* { dg-final { scan-tree-dump "add new stmt: \[^\n\r]*COMPLEX_FMS \\(" "vect" { xfail arm*-*-* } } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump-not "Found COMPLEX_MUL" "vect" } } */
diff --git a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90 b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
index b06950d1baae7d8aca2e1c96c8e6e942781b71bd..d62330d4a7760235bf25179b072d986bbe87c653 100644
--- a/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
+++ b/gcc/testsuite/gfortran.target/aarch64/pr122408_1.f90
@@ -57,10 +57,10 @@ subroutine c_sub_a_conjb(n, a, c, b) ! C -= A * conj(B)
end do
end subroutine c_sub_a_conjb
-! The two plain multiplies form .COMPLEX_MUL (#0 + #90) and the two conjugate
-! multiplies form .COMPLEX_MUL_CONJ (#0 + #270). PR122408 is about detecting
-! the conjugate form, so the #270 and #180 counts are the ones that guard it.
-! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #0} 4 } }
+! The accumulations form .COMPLEX_FMA (#0 + #90) and .COMPLEX_FMS (#180 +
+! #270), with the conjugate forms swapping the #90 and #270 rotations. PR122408
+! is about detecting the conjugate form, so the #90 and #270 counts guard it.
+! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #0} 2 } }
! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #90} 2 } }
! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #270} 2 } }
-! { dg-final { scan-assembler-not {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #180} } }
+! { dg-final { scan-assembler-times {fcmla\s+v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d, #180} 2 } }
diff --git a/gcc/tree-vect-slp-patterns.cc b/gcc/tree-vect-slp-patterns.cc
index f77993100ad19ce5502900022a68b39201d24a54..aae000ee5292ccb1521928e43f514a14386a7827 100644
--- a/gcc/tree-vect-slp-patterns.cc
+++ b/gcc/tree-vect-slp-patterns.cc
@@ -1305,7 +1305,94 @@ complex_fms_pattern::matches (complex_operation_t op,
slp_compat_nodes_map_t *compat_cache,
slp_tree * ref_node, vec<slp_tree> *ops)
{
- internal_fn ifn = IFN_LAST;
+ /* It's only valid to form FMSs with -ffp-contract=fast. */
+ if (!SLP_TREE_VECTYPE (*ref_node)
+ || (flag_fp_contract_mode != FP_CONTRACT_FAST
+ && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node))))
+ return IFN_LAST;
+
+ /* Match c - a * b when SLP has built the result as:
+
+ c.real + (a.imag * b.imag - a.real * b.real)
+ c.imag - (a.real * b.imag + a.imag * b.real)
+
+ This represents the same operation as the existing FMS matcher below,
+ but with the accumulator outside the complex product node. */
+ if (op == PLUS_MINUS)
+ {
+ auto plus_ops = SLP_TREE_CHILDREN ((*ops)[0]);
+ auto minus_ops = SLP_TREE_CHILDREN ((*ops)[1]);
+ if (plus_ops.length () != 2 || minus_ops.length () != 2)
+ return IFN_LAST;
+
+ slp_tree acc = minus_ops[0];
+ slp_tree prod = minus_ops[1];
+ if (!((plus_ops[0] == acc && plus_ops[1] == prod)
+ || (plus_ops[1] == acc && plus_ops[0] == prod)))
+ return IFN_LAST;
+ if (linear_loads_p (perm_cache, acc) != PERM_EVENODD)
+ return IFN_LAST;
+
+ auto_vec<slp_tree> prod_ops;
+ if (vect_detect_pair_op (prod, true, &prod_ops) != MINUS_PLUS)
+ return IFN_LAST;
+ if (prod_ops.length () != 2)
+ return IFN_LAST;
+
+ auto prod_left = SLP_TREE_CHILDREN (prod_ops[0]);
+ auto prod_right = SLP_TREE_CHILDREN (prod_ops[1]);
+ if (prod_left.length () != 2
+ || prod_right.length () != 2
+ || !vect_match_expression_p (prod_left[0], MULT_EXPR)
+ || !vect_match_expression_p (prod_left[1], MULT_EXPR)
+ || !vect_match_expression_p (prod_right[0], MULT_EXPR)
+ || !vect_match_expression_p (prod_right[1], MULT_EXPR))
+ return IFN_LAST;
+
+ auto_vec<slp_tree> left_op, right_op;
+ left_op.safe_splice (SLP_TREE_CHILDREN (prod_left[0]));
+ right_op.safe_splice (SLP_TREE_CHILDREN (prod_left[1]));
+
+ enum _conj_status status;
+ auto_vec<slp_tree> res_ops;
+ if (!vect_validate_multiplication_commutative (perm_cache, compat_cache,
+ right_op, left_op, true,
+ res_ops, &status))
+ return IFN_LAST;
+
+ internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS
+ : IFN_COMPLEX_FMS_CONJ;
+ if (!vect_pattern_validate_optab (ifn, *ref_node))
+ return IFN_LAST;
+
+ ops->truncate (0);
+ ops->create (4);
+
+ complex_perm_kinds_t kind = linear_loads_p (perm_cache, res_ops[0]);
+ if (kind == PERM_EVENODD || kind == PERM_TOP)
+ {
+ ops->quick_push (acc);
+ ops->quick_push (res_ops[0]);
+ ops->quick_push (res_ops[1]);
+ ops->quick_push (res_ops[3]);
+ }
+ else if (kind == PERM_EVENEVEN && status != CONJ_SND)
+ {
+ ops->quick_push (acc);
+ ops->quick_push (res_ops[1]);
+ ops->quick_push (res_ops[0]);
+ ops->quick_push (res_ops[2]);
+ }
+ else
+ {
+ ops->quick_push (acc);
+ ops->quick_push (res_ops[1]);
+ ops->quick_push (res_ops[0]);
+ ops->quick_push (res_ops[3]);
+ }
+
+ return ifn;
+ }
/* We need to ignore the two_operands nodes that may also match,
for that we can check if they have any scalar statements and also
@@ -1318,11 +1405,6 @@ complex_fms_pattern::matches (complex_operation_t op,
if (!vect_match_expression_p (root, MINUS_EXPR))
return IFN_LAST;
- /* It's only valid to form FMSs with -ffp-contract=fast. */
- if (flag_fp_contract_mode != FP_CONTRACT_FAST
- && FLOAT_TYPE_P (SLP_TREE_VECTYPE (*ref_node)))
- return IFN_LAST;
-
/* TODO: Support invariants here, with the new layout CADD now
can match before we get a chance to try CFMS. */
auto nodes = SLP_TREE_CHILDREN (root);
@@ -1352,11 +1434,8 @@ complex_fms_pattern::matches (complex_operation_t op,
res_ops, &status))
return IFN_LAST;
- if (status == CONJ_NONE)
- ifn = IFN_COMPLEX_FMS;
- else
- ifn = IFN_COMPLEX_FMS_CONJ;
-
+ internal_fn ifn = status == CONJ_NONE ? IFN_COMPLEX_FMS
+ : IFN_COMPLEX_FMS_CONJ;
if (!vect_pattern_validate_optab (ifn, *ref_node))
return IFN_LAST;
