Ports the NEON fused-multiple-add (vfma) and fused-multiply-subtract (vfms)
intrinsics to the pragma-based framework, and adds corresponding
assembly tests for each.
Some fnma insns had to have their patterns adjusted to match the RTL
produced by the FNMA IFN (the negated input should always be the first
input operand). Also reformat the insns for consistency.
Also added comments above each fma/fnma insn explaining which intrinsics
they cover, because there are 3 different insns for each intrinsic and
it can be non-obvious which intrinsic maps to which insn (especially
insns that take `float64x1_t` or `float64_t` as arguments).
The `__builtin_aarch64_fma` and `__builtin_aarch64_fnma` builtin
functions were deleted because they are no longer used - except for the
scalar HF variants, which are still used by `arm_f16.h`
gcc/ChangeLog:
* config/aarch64/aarch64-neon-builtins-base.cc
(build_ifn_call): New function.
(gimple_ifn::fold): Call `build_ifn_call`.
(struct gimple_fused_mla): New struct.
* config/aarch64/aarch64-neon-builtins-base.def
(vfma, vfmaq, vfma_n, vfmaq_n, vfma_lane, vfma_laneq,
vfmaq_lane, vfmaq_laneq, vfmas_lane, vfmas_laneq, vfmad_lane,
vfmad_laneq, vfms, vfmsq, vfms_n, vfmsq_n, vfms_lane,
vfms_laneq, vfmsq_lane, vfmsq_laneq, vfmss_lane, vfmss_laneq,
vfmsd_lane, vfmsd_laneq, and the FP16 forms of each): New
intrinsic function declarations.
* config/aarch64/aarch64-simd-builtins.def
(fma, fnma): Delete `__builtin_aarch64` function declarations.
* config/aarch64/aarch64-simd.md
(fma<mode>4<vczle><vczbe>,
*aarch64_fma4_elt<mode><vczle><vczbe>,
*aarch64_fma4_elt_<vswap_width_name><mode><vczle><vczbe>,
*aarch64_fma4_elt_from_dup<mode><vczle><vczbe>,
*aarch64_fma<mode>4_elt_to_scalar): Extend to cover HF vector
modes, reformat, add comments.
(fnma<mode>4<vczle><vczbe>,
*aarch64_fnma4_elt<mode><vczle><vczbe>
*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>
*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>
*aarch64_fnma<mode>4_elt_from_to_scalar):
Likewise.
(*aarch64_fma4_elt_to_64v2df, *aarch64_fnma4_elt_to_64v2df):
Rename to ...
(*aarch64_fma<mode>4_elt_to_scalar,
*aarch64_fnma<mode>4_elt_to_scalar): ...these.
(*aarch64_fnma4_elt<mode><vczle><vczbe>_alt,
*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>_alt,
*aarch64_fnma4_elt_from_dup<mode><cvzle><vczbe>_alt,
*aarch64_fnma4<mode>4_elt_to_scalar_alt): New insns to match
permuted variants.
* config/aarch64/arm_neon.h
(vfma_f64, vfma_f32, vfmaq_f32, vfmaq_f64, vfma_n_f32,
vfma_n_f64, vfmaq_n_f32, vfmaq_n_f64, vfma_lane_f32,
vfma_lane_f64, vfma_laneq_f32, vfma_laneq_f64,
vfmaq_lane_f32, vfmaq_lane_f64, vfmaq_laneq_f32,
vfmaq_laneq_f64, vfms_f64, vfms_f32, vfmsq_f32, vfmsq_f64,
vfms_n_f32, vfms_n_f64, vfmsq_n_f32, vfmsq_n_f64,
vfms_lane_f32, vfms_lane_f64, vfms_laneq_f32, vfms_laneq_f64,
vfmsq_lane_f32, vfmsq_lane_f64, vfmsq_laneq_f32,
vfmsq_laneq_f64, vfma_f16, vfmaq_f16, vfms_f16, vfmsq_f16,
vfma_lane_f16, vfmaq_lane_f16, vfma_laneq_f16,
vfmaq_laneq_f16, vfma_n_f16, vfmaq_n_f16, vfms_lane_f16,
vfmsq_lane_f16, vfms_laneq_f16, vfmsq_laneq_f16, vfms_n_f16,
vfmsq_n_f16): Delete function definitions.
* config/aarch64/iterators.md (VDQHSF): New mode iterator.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/fmla_intrinsic_1.c: Rewrite `TEST_VMLA`
so the lane choice is always valid and doesn't cause a compiler
error when trying to call `vfma`. Also remove excessive
backslashes from dejagnu directives.
* gcc.target/aarch64/fmls_intrinsic_1.c: Likewise.
* gcc.target/aarch64/simd/pr99195_3.c: Add
`-fno-trapping-math` to prevent the optimization barrier being
introduced and generating unnecessary `fmov`s.
* gcc.target/aarch64/neon/vfma.c: New test.
* gcc.target/aarch64/neon/vfms.c: New test.
---
.../aarch64/aarch64-neon-builtins-base.cc | 122 ++++-
.../aarch64/aarch64-neon-builtins-base.def | 54 ++
gcc/config/aarch64/aarch64-simd-builtins.def | 2 -
gcc/config/aarch64/aarch64-simd.md | 244 ++++++---
gcc/config/aarch64/arm_neon.h | 500 ------------------
gcc/config/aarch64/iterators.md | 7 +
.../gcc.target/aarch64/fmla_intrinsic_1.c | 25 +-
.../gcc.target/aarch64/fmls_intrinsic_1.c | 25 +-
gcc/testsuite/gcc.target/aarch64/neon/vfma.c | 220 ++++++++
gcc/testsuite/gcc.target/aarch64/neon/vfms.c | 220 ++++++++
.../gcc.target/aarch64/simd/pr99195_3.c | 3 +-
11 files changed, 780 insertions(+), 642 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vfma.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vfms.c
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
index bdb14273f5f..f96b3332dc4 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
@@ -176,6 +176,36 @@ one_element_vector_type_p (tree type)
return VECTOR_TYPE_P (type) && known_eq (TYPE_VECTOR_SUBPARTS (type), 1U);
}
+/* Build a call to the internal function IFN.
+ Handles singleton vector arguments specially: they are cast to their
+ inner element type before the function is called, and then the return value
+ is cast back to the expected singleton vector type. */
+tree
+build_ifn_call (gimple_folder &f, internal_fn ifn, tree ret_type,
+ vec<tree> &args)
+{
+ for (auto &arg : args)
+ if (one_element_vector_type_p (TREE_TYPE (arg)))
+ arg = f.force_val (build_cast (TREE_TYPE (TREE_TYPE (arg)), arg));
+
+ auto call = gimple_build_call_internal_vec (ifn, args);
+ gimple_seq_add_stmt_without_update (&f.stmts, call);
+
+ if (!one_element_vector_type_p (ret_type))
+ {
+ auto ret = is_gimple_reg_type (ret_type) ? make_ssa_name (ret_type)
+ : create_tmp_var (ret_type);
+ gimple_call_set_lhs (call, ret);
+ return ret;
+ }
+ else
+ {
+ auto ret = create_tmp_var (TREE_TYPE (ret_type));
+ gimple_call_set_lhs (call, ret);
+ return build_vec_dup (ret_type, ret);
+ }
+}
+
/* Base class for all function expanders.
At least one of `expand` or `fold` must be overriden by derived classes. */
class gimple_function_base : public function_base
@@ -644,44 +674,19 @@ public:
gimple *fold (gimple_folder &f) const override
{
tree lhs_type = TREE_TYPE (f.lhs);
- /* There are no instructions that handle V1DI or V1DF modes for these
- operations. They need to be specially handled by casting the vector to
- its element type and using it to feed the scalar version of the
- operation. The scalar result is then cast back to a vector and
returned.
- This matches the prior implementation that arm_neon.h used for these
- operations. */
- bool scalar_lhs = one_element_vector_type_p (lhs_type);
-
vec<tree> args{};
for (unsigned i = 0; i < gimple_call_num_args (f.call); i++)
{
tree arg = gimple_call_arg (f.call, i);
- /* Convert V1DI/V1DF arguments to corresponding scalar modes. */
- if (scalar_lhs && one_element_vector_type_p (TREE_TYPE (arg)))
- arg = f.force_val (build_cast (TREE_TYPE (lhs_type), arg));
/* .CLZ and .POPCOUNT take an unsigned argument. */
- else if ((m_ifn == IFN_CLZ || m_ifn == IFN_POPCOUNT)
- && !TYPE_UNSIGNED (TREE_TYPE (arg)))
+ if ((m_ifn == IFN_CLZ || m_ifn == IFN_POPCOUNT)
+ && !TYPE_UNSIGNED (TREE_TYPE (arg)))
arg = f.force_val (build_cast (unsigned_type_for (TREE_TYPE (arg)),
arg));
args.safe_push (arg);
}
-
- auto call = gimple_build_call_internal_vec (this->m_ifn, args);
- args.release ();
-
- if (!scalar_lhs)
- {
- gimple_call_set_lhs (call, f.lhs);
- return call;
- }
-
- /* Temporary scalar to hold the result. */
- tree res = create_tmp_var (TREE_TYPE (lhs_type));
- gimple_call_set_lhs (call, res);
- gimple_seq_add_stmt_without_update (&f.stmts, call);
- /* Build the actual vector that the call's result gets assigned to. */
- return gimple_build_assign (f.lhs, build_vec_dup (lhs_type, res));
+ return gimple_build_assign (f.lhs, build_ifn_call (f, this->m_ifn,
lhs_type,
+ args));
}
};
@@ -1019,6 +1024,46 @@ struct multiply_accumulate : public gimple_function_base
template <arith_kind kind> using gimple_mla = multiply_accumulate<kind, false>;
template <arith_kind kind> using gimple_mls = multiply_accumulate<kind, true>;
+template <arith_kind kind, bool negate>
+struct gimple_fused_mla : public gimple_function_base
+{
+ gimple *fold (gimple_folder &f) const override
+ {
+ tree acc = gimple_call_arg (f.call, 0);
+ tree lhs = gimple_call_arg (f.call, 1);
+ tree rhs = gimple_call_arg (f.call, 2);
+ tree lane_index;
+ tree ret_type = TREE_TYPE (f.lhs);
+
+ switch (kind)
+ {
+ case arith_kind::normal:
+ break;
+
+ case arith_kind::dup:
+ rhs = f.force_val (build_vec_dup (TREE_TYPE (lhs), rhs));
+ break;
+
+ case arith_kind::lane:
+ lane_index = gimple_call_arg (f.call, 3);
+ rhs = f.force_val (build_lane_get (rhs, lane_index));
+ if (VECTOR_TYPE_P (ret_type))
+ rhs = f.force_val (build_vec_dup (TREE_TYPE (lhs), rhs));
+ break;
+ }
+
+ vec<tree> args{};
+ args.safe_push (lhs);
+ args.safe_push (rhs);
+ args.safe_push (acc);
+ auto ret = build_ifn_call (f, negate ? IFN_FNMA : IFN_FMA, ret_type, args);
+ args.release ();
+ return this->maybe_optimization_barrier (f, ret);
+ }
+};
+template <arith_kind kind> using gimple_fma = gimple_fused_mla<kind, false>;
+template <arith_kind kind> using gimple_fms = gimple_fused_mla<kind, true>;
+
// Reinterpret
NEON_FUNCTION (vreinterpret, vreinterpretq, gimple_reinterpret,)
@@ -1060,6 +1105,25 @@ NEON_FUNCTION (vmls_n, vmlsq_n,
gimple_mls<arith_kind::dup>,)
NEON_FUNCTION (vmls_lane, vmls_laneq, vmlsq_lane, vmlsq_laneq,
gimple_mls<arith_kind::lane>,)
+// Multiply-accumulate (fused)
+NEON_FUNCTION (vfma, vfmaq, gimple_fma<arith_kind::normal>,);
+NEON_FUNCTION (vfma_n, vfmaq_n, gimple_fma<arith_kind::dup>,);
+NEON_FUNCTION (vfmah_lane, vfmah_laneq,
+ vfmas_lane, vfmas_laneq,
+ vfmad_lane, vfmad_laneq,
+ vfma_lane, vfma_laneq,
+ vfmaq_lane, vfmaq_laneq,
+ gimple_fma<arith_kind::lane>,);
+
+NEON_FUNCTION (vfms, vfmsq, gimple_fms<arith_kind::normal>,);
+NEON_FUNCTION (vfms_n, vfmsq_n, gimple_fms<arith_kind::dup>,);
+NEON_FUNCTION (vfmsh_lane, vfmsh_laneq,
+ vfmss_lane, vfmss_laneq,
+ vfmsd_lane, vfmsd_laneq,
+ vfms_lane, vfms_laneq,
+ vfmsq_lane, vfmsq_laneq,
+ gimple_fms<arith_kind::lane>,);
+
// Saturating arithmetic
NEON_FUNCTION (vqaddb, vqaddh, vqadds, vqaddd, vqadd, vqaddq, gimple_ifn,
(IFN_SAT_ADD))
NEON_FUNCTION (vqsubb, vqsubh, vqsubs, vqsubd, vqsub, vqsubq, gimple_ifn,
(IFN_SAT_SUB))
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.def
b/gcc/config/aarch64/aarch64-neon-builtins-base.def
index 40aa1a656d7..e519e47beb2 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.def
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.def
@@ -296,6 +296,60 @@ DEF_NEON_FUNCTION (vmlsq_lane, neon_mla_lane,
("Q0,Q0,Q0,D0,ss32", lane<3>))
DEF_NEON_FUNCTION (vmlsq_laneq, neon_mla_lane, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
#undef REQUIRED_EXTENSIONS
+// Fused multiply-accumulate
+#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
+DEF_NEON_FUNCTION (vfma, sd_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmaq, sd_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfma_n, sd_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmaq_n, sd_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfma_lane, sd_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfma_laneq, sd_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_lane, sd_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_laneq, sd_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmas_lane, s_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmas_laneq, s_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmad_lane, d_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmad_laneq, d_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+
+DEF_NEON_FUNCTION (vfms, sd_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmsq, sd_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfms_n, sd_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmsq_n, sd_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfms_lane, sd_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfms_laneq, sd_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_lane, sd_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_laneq, sd_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmss_lane, s_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmss_laneq, s_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsd_lane, d_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsd_laneq, d_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+#undef REQUIRED_EXTENSIONS
+
+// Fused multiply-accumulate (FP16)
+#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD |
AARCH64_FL_F16)
+DEF_NEON_FUNCTION (vfma, h_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmaq, h_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfma_n, h_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmaq_n, h_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfma_lane, h_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfma_laneq, h_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_lane, h_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_laneq, h_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmah_lane, h_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmah_laneq, h_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+
+DEF_NEON_FUNCTION (vfms, h_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmsq, h_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfms_n, h_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmsq_n, h_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfms_lane, h_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfms_laneq, h_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_lane, h_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_laneq, h_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsh_lane, h_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsh_laneq, h_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+#undef REQUIRED_EXTENSIONS
+
// Bitwise operations
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
DEF_NEON_FUNCTION (vand, all_integer, ("D0,D0,D0"))
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def
b/gcc/config/aarch64/aarch64-simd-builtins.def
index 1fb7c763990..3792ec3edf4 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -669,10 +669,8 @@
BUILTIN_VALLP_NO_DI (STORE1_P, st1, 0, STORE)
/* Implemented by fma<mode>4. */
- BUILTIN_VHSDF (TERNOP, fma, 4, FP)
VAR1 (TERNOP, fma, 4, FP, hf)
/* Implemented by fnma<mode>4. */
- BUILTIN_VHSDF (TERNOP, fnma, 4, FP)
VAR1 (TERNOP, fnma, 4, FP, hf)
/* Implemented by aarch64_crypto_pmull<mode>. */
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index a42e1a60c58..7cc0d97b7c3 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -2962,9 +2962,9 @@ (define_expand "aarch64_<su>mlsl_hi_laneq<mode>"
)
;; FP vector operations.
-;; AArch64 AdvSIMD supports single-precision (32-bit) and
+;; AArch64 AdvSIMD supports single-precision (32-bit) and
;; double-precision (64-bit) floating-point data types and arithmetic as
-;; defined by the IEEE 754-2008 standard. This makes them vectorizable
+;; defined by the IEEE 754-2008 standard. This makes them vectorizable
;; without the need for -ffast-math or -funsafe-math-optimizations.
;;
;; Floating-point operations can raise an exception. Vectorizing such
@@ -3059,78 +3059,95 @@ (define_insn "abs<mode>2<vczle><vczbe>"
[(set_attr "type" "neon_fp_abs_<stype><q>")]
)
+;; vfma_f16, vfmaq_f16
+;; vfma_f32, vfmaq_f32
+;; vfmaq_f64
(define_insn "fma<mode>4<vczle><vczbe>"
[(set (match_operand:VHSDF 0 "register_operand" "=w")
- (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
- (match_operand:VHSDF 2 "register_operand" "w")
- (match_operand:VHSDF 3 "register_operand" "0")))]
+ (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
+ (match_operand:VHSDF 2 "register_operand" "w")
+ (match_operand:VHSDF 3 "register_operand" "0")))]
"TARGET_SIMD"
"fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
[(set_attr "type" "neon_fp_mla_<stype><q>")]
)
+;; vfma_lane_f16, vfmaq_laneq_f16
+;; vfma_lane_f32, vfmaq_laneq_f32
+;; vfmaq_laneq_f64
(define_insn "*aarch64_fma4_elt<mode><vczle><vczbe>"
- [(set (match_operand:VDQF 0 "register_operand" "=w")
- (fma:VDQF
- (vec_duplicate:VDQF
- (vec_select:<VEL>
- (match_operand:VDQF 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQF 3 "register_operand" "w")
- (match_operand:VDQF 4 "register_operand" "0")))]
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (vec_duplicate:VHSDF
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VHSDF 3 "register_operand" "w")
+ (match_operand:VHSDF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfma_laneq_f16, vfmaq_lane_f16
+;; vfma_laneq_f32, vfmaq_lane_f32
(define_insn "*aarch64_fma4_elt_<vswap_width_name><mode><vczle><vczbe>"
- [(set (match_operand:VDQSF 0 "register_operand" "=w")
- (fma:VDQSF
- (vec_duplicate:VDQSF
- (vec_select:<VEL>
- (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQSF 3 "register_operand" "w")
- (match_operand:VDQSF 4 "register_operand" "0")))]
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (vec_duplicate:VDQHSF
+ (vec_select:<VEL>
+ (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VDQHSF 3 "register_operand" "w")
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfma_n_f16, vfmaq_n_f16
+;; vfma_n_f32, vfmaq_n_f64
+;; vfmaq_n_f64 vfmaq_lane_f64
(define_insn "*aarch64_fma4_elt_from_dup<mode><vczle><vczbe>"
[(set (match_operand:VMUL 0 "register_operand" "=w")
- (fma:VMUL
- (vec_duplicate:VMUL
- (match_operand:<VEL> 1 "register_operand" "<h_con>"))
- (match_operand:VMUL 2 "register_operand" "w")
- (match_operand:VMUL 3 "register_operand" "0")))]
+ (fma:VMUL
+ (vec_duplicate:VMUL
+ (match_operand:<VEL> 1 "register_operand" "<h_con>"))
+ (match_operand:VMUL 2 "register_operand" "w")
+ (match_operand:VMUL 3 "register_operand" "0")))]
"TARGET_SIMD"
"fmla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
[(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
)
-(define_insn "*aarch64_fma4_elt_to_64v2df"
- [(set (match_operand:DF 0 "register_operand" "=w")
- (fma:DF
- (vec_select:DF
- (match_operand:V2DF 1 "register_operand" "w")
- (parallel [(match_operand:SI 2 "immediate_operand")]))
- (match_operand:DF 3 "register_operand" "w")
- (match_operand:DF 4 "register_operand" "0")))]
+;; vfmah_lane_f16, vfmah_laneq_f16
+;; vfmas_lane_f32, vfmas_laneq_f32
+;; vfma_laneq_f64, vfmad_laneq_f64
+(define_insn "*aarch64_fma<mode>4_elt_to_scalar"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))
+ (match_operand:<VEL> 3 "register_operand" "w")
+ (match_operand:<VEL> 4 "register_operand" "0")))]
"TARGET_SIMD"
{
- operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
- return "fmla\\t%0.2d, %3.2d, %1.d[%2]";
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmla\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_d_scalar_q")]
+ [(set_attr "type" "neon_fp_mla_<stype>_q")]
)
+;; vfms_f16, vfmsq_f16
+;; vfms_f32, vfmsq_f32
+;; vfmsq_f64
(define_insn "fnma<mode>4<vczle><vczbe>"
[(set (match_operand:VHSDF 0 "register_operand" "=w")
(fma:VHSDF
@@ -3142,70 +3159,143 @@ (define_insn "fnma<mode>4<vczle><vczbe>"
[(set_attr "type" "neon_fp_mla_<stype><q>")]
)
+;; vfms_lane_f16, vfmsq_laneq_f16,
+;; vfms_lane_f32, vfmsq_laneq_f32
+;; vfmsq_laneq_f64
(define_insn "*aarch64_fnma4_elt<mode><vczle><vczbe>"
- [(set (match_operand:VDQF 0 "register_operand" "=w")
- (fma:VDQF
- (neg:VDQF
- (match_operand:VDQF 3 "register_operand" "w"))
- (vec_duplicate:VDQF
- (vec_select:<VEL>
- (match_operand:VDQF 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQF 4 "register_operand" "0")))]
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (neg:VHSDF
+ (vec_duplicate:VHSDF
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))))
+ (match_operand:VHSDF 3 "register_operand" "w")
+ (match_operand:VHSDF 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ {
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
+ }
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
+)
+(define_insn "*aarch64_fnma4_elt<mode><vczle><vczbe>_alt"
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (neg:VHSDF
+ (match_operand:VHSDF 3 "register_operand" "w"))
+ (vec_duplicate:VHSDF
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VHSDF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfms_laneq_f16, vfmsq_lane_f16
+;; vfms_laneq_f32, vfmsq_lane_f32
(define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>"
- [(set (match_operand:VDQSF 0 "register_operand" "=w")
- (fma:VDQSF
- (neg:VDQSF
- (match_operand:VDQSF 3 "register_operand" "w"))
- (vec_duplicate:VDQSF
- (vec_select:<VEL>
- (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQSF 4 "register_operand" "0")))]
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (neg:VDQHSF
+ (vec_duplicate:VDQHSF
+ (vec_select:<VEL>
+ (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))))
+ (match_operand:VDQHSF 3 "register_operand" "w")
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ {
+ operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
+ return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
+ }
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
+)
+(define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>_alt"
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (neg:VDQHSF
+ (match_operand:VDQHSF 3 "register_operand" "w"))
+ (vec_duplicate:VDQHSF
+ (vec_select:<VEL>
+ (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfms_n_f16, vfmsq_n_f16
+;; vfms_n_f32, vfmsq_n_f32
+;; vfmsq_n_f64, vfmq_lane_f64
(define_insn "*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>"
[(set (match_operand:VMUL 0 "register_operand" "=w")
- (fma:VMUL
- (neg:VMUL
- (match_operand:VMUL 2 "register_operand" "w"))
- (vec_duplicate:VMUL
- (match_operand:<VEL> 1 "register_operand" "<h_con>"))
- (match_operand:VMUL 3 "register_operand" "0")))]
+ (fma:VMUL
+ (neg:VMUL
+ (vec_duplicate:VMUL
+ (match_operand:<VEL> 1 "register_operand" "<h_con>")))
+ (match_operand:VMUL 2 "register_operand" "w")
+ (match_operand:VMUL 3 "register_operand" "0")))]
+ "TARGET_SIMD"
+ "fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
+ [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
+)
+(define_insn "*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>_alt"
+ [(set (match_operand:VMUL 0 "register_operand" "=w")
+ (fma:VMUL
+ (neg:VMUL
+ (match_operand:VMUL 2 "register_operand" "w"))
+ (vec_duplicate:VMUL
+ (match_operand:<VEL> 1 "register_operand" "<h_con>"))
+ (match_operand:VMUL 3 "register_operand" "0")))]
"TARGET_SIMD"
"fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
[(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
)
-(define_insn "*aarch64_fnma4_elt_to_64v2df"
- [(set (match_operand:DF 0 "register_operand" "=w")
- (fma:DF
- (vec_select:DF
- (match_operand:V2DF 1 "register_operand" "w")
- (parallel [(match_operand:SI 2 "immediate_operand")]))
- (neg:DF
- (match_operand:DF 3 "register_operand" "w"))
- (match_operand:DF 4 "register_operand" "0")))]
+;; vfmsh_lane_f16, vfmsh_laneq_f16
+;; vfmss_lane_f32, vfmss_laneq_f32
+;; vfms_laneq_f64, vfmsd_laneq_f64
+(define_insn "*aarch64_fnma<mode>4_elt_to_scalar"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (neg:<VEL>
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:<VEL> 3 "register_operand" "w")
+ (match_operand:<VEL> 4 "register_operand" "0")))]
"TARGET_SIMD"
{
- operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
- return "fmls\\t%0.2d, %3.2d, %1.d[%2]";
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmls\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]";
+ }
+ [(set_attr "type" "neon_fp_mla_<stype>_q")]
+)
+(define_insn "*aarch64_fnma<mode>4_elt_to_scalar_alt"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (neg:<VEL>
+ (match_operand:<VEL> 3 "register_operand" "w"))
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))
+ (match_operand:<VEL> 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ {
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmls\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_d_scalar_q")]
+ [(set_attr "type" "neon_fp_mla_<stype>_q")]
)
;; Vector versions of the floating-point frint patterns.
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index 71b93a6adf5..db9d5c4c185 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -6858,352 +6858,6 @@ vcvtpq_u64_f64 (float64x2_t __a)
return __builtin_aarch64_lceiluv2dfv2di_us (__a);
}
-/* vfma */
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f64 (float64x1_t __a, float64x1_t __b, float64x1_t __c)
-{
- return (float64x1_t) {__builtin_fma (__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f32 (float32x2_t __a, float32x2_t __b, float32x2_t __c)
-{
- return __builtin_aarch64_fmav2sf (__b, __c, __a);
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f32 (float32x4_t __a, float32x4_t __b, float32x4_t __c)
-{
- return __builtin_aarch64_fmav4sf (__b, __c, __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f64 (float64x2_t __a, float64x2_t __b, float64x2_t __c)
-{
- return __builtin_aarch64_fmav2df (__b, __c, __a);
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f32 (float32x2_t __a, float32x2_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav2sf (__b, vdup_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f64 (float64x1_t __a, float64x1_t __b, float64_t __c)
-{
- return (float64x1_t) {__b[0] * __c + __a[0]};
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f32 (float32x4_t __a, float32x4_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav4sf (__b, vdupq_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f64 (float64x2_t __a, float64x2_t __b, float64_t __c)
-{
- return __builtin_aarch64_fmav2df (__b, vdupq_n_f64 (__c), __a);
-}
-
-/* vfma_lane */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f32 (float32x2_t __a, float32x2_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (__b,
- __aarch64_vdup_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f64 (float64x1_t __a, float64x1_t __b,
- float64x1_t __c, const int __lane)
-{
- return (float64x1_t) {__builtin_fma (__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmad_lane_f64 (float64_t __a, float64_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_fma (__b, __c[0], __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmas_lane_f32 (float32_t __a, float32_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_fmaf (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfma_laneq */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f32 (float32x2_t __a, float32x2_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (__b,
- __aarch64_vdup_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f64 (float64x1_t __a, float64x1_t __b,
- float64x2_t __c, const int __lane)
-{
- float64_t __c0 = __aarch64_vget_lane_any (__c, __lane);
- return (float64x1_t) {__builtin_fma (__b[0], __c0, __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmad_laneq_f64 (float64_t __a, float64_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_fma (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmas_laneq_f32 (float32_t __a, float32_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_fmaf (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfmaq_lane */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f32 (float32x4_t __a, float32x4_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (__b,
- __aarch64_vdupq_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f64 (float64x2_t __a, float64x2_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (__b, vdupq_n_f64 (__c[0]), __a);
-}
-
-/* vfmaq_laneq */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f32 (float32x4_t __a, float32x4_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (__b,
- __aarch64_vdupq_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f64 (float64x2_t __a, float64x2_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (__b,
- __aarch64_vdupq_laneq_f64 (__c, __lane),
- __a);
-}
-
-/* vfms */
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f64 (float64x1_t __a, float64x1_t __b, float64x1_t __c)
-{
- return (float64x1_t) {__builtin_fma (-__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f32 (float32x2_t __a, float32x2_t __b, float32x2_t __c)
-{
- return __builtin_aarch64_fmav2sf (-__b, __c, __a);
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f32 (float32x4_t __a, float32x4_t __b, float32x4_t __c)
-{
- return __builtin_aarch64_fmav4sf (-__b, __c, __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f64 (float64x2_t __a, float64x2_t __b, float64x2_t __c)
-{
- return __builtin_aarch64_fmav2df (-__b, __c, __a);
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f32 (float32x2_t __a, float32x2_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav2sf (-__b, vdup_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f64 (float64x1_t __a, float64x1_t __b, float64_t __c)
-{
- return (float64x1_t) {-__b[0] * __c + __a[0]};
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f32 (float32x4_t __a, float32x4_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav4sf (-__b, vdupq_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f64 (float64x2_t __a, float64x2_t __b, float64_t __c)
-{
- return __builtin_aarch64_fmav2df (-__b, vdupq_n_f64 (__c), __a);
-}
-
-/* vfms_lane */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f32 (float32x2_t __a, float32x2_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (-__b,
- __aarch64_vdup_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f64 (float64x1_t __a, float64x1_t __b,
- float64x1_t __c, const int __lane)
-{
- return (float64x1_t) {__builtin_fma (-__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsd_lane_f64 (float64_t __a, float64_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_fma (-__b, __c[0], __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmss_lane_f32 (float32_t __a, float32_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_fmaf (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfms_laneq */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f32 (float32x2_t __a, float32x2_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (-__b,
- __aarch64_vdup_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f64 (float64x1_t __a, float64x1_t __b,
- float64x2_t __c, const int __lane)
-{
- float64_t __c0 = __aarch64_vget_lane_any (__c, __lane);
- return (float64x1_t) {__builtin_fma (-__b[0], __c0, __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsd_laneq_f64 (float64_t __a, float64_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_fma (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmss_laneq_f32 (float32_t __a, float32_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_fmaf (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfmsq_lane */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f32 (float32x4_t __a, float32x4_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (-__b,
- __aarch64_vdupq_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f64 (float64x2_t __a, float64x2_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (-__b, vdupq_n_f64 (__c[0]), __a);
-}
-
-/* vfmsq_laneq */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f32 (float32x4_t __a, float32x4_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (-__b,
- __aarch64_vdupq_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f64 (float64x2_t __a, float64x2_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (-__b,
- __aarch64_vdupq_laneq_f64 (__c, __lane),
- __a);
-}
-
/* vld1 */
__extension__ extern __inline float16x4_t
@@ -17260,162 +16914,8 @@ vrsqrtsq_f16 (float16x8_t __a, float16x8_t __b)
return __builtin_aarch64_rsqrtsv8hf (__a, __b);
}
-/* ARMv8.2-A FP16 three operands vector intrinsics. */
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f16 (float16x4_t __a, float16x4_t __b, float16x4_t __c)
-{
- return __builtin_aarch64_fmav4hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f16 (float16x8_t __a, float16x8_t __b, float16x8_t __c)
-{
- return __builtin_aarch64_fmav8hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f16 (float16x4_t __a, float16x4_t __b, float16x4_t __c)
-{
- return __builtin_aarch64_fnmav4hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f16 (float16x8_t __a, float16x8_t __b, float16x8_t __c)
-{
- return __builtin_aarch64_fnmav8hf (__b, __c, __a);
-}
-
/* ARMv8.2-A FP16 lane vector intrinsics. */
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmah_lane_f16 (float16_t __a, float16_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmah_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmah_laneq_f16 (float16_t __a, float16_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmah_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f16 (float16x4_t __a, float16x4_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfma_f16 (__a, __b, __aarch64_vdup_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f16 (float16x8_t __a, float16x8_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmaq_f16 (__a, __b, __aarch64_vdupq_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f16 (float16x4_t __a, float16x4_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfma_f16 (__a, __b, __aarch64_vdup_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f16 (float16x8_t __a, float16x8_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmaq_f16 (__a, __b, __aarch64_vdupq_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f16 (float16x4_t __a, float16x4_t __b, float16_t __c)
-{
- return vfma_f16 (__a, __b, vdup_n_f16 (__c));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f16 (float16x8_t __a, float16x8_t __b, float16_t __c)
-{
- return vfmaq_f16 (__a, __b, vdupq_n_f16 (__c));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsh_lane_f16 (float16_t __a, float16_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmsh_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsh_laneq_f16 (float16_t __a, float16_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmsh_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f16 (float16x4_t __a, float16x4_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfms_f16 (__a, __b, __aarch64_vdup_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f16 (float16x8_t __a, float16x8_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmsq_f16 (__a, __b, __aarch64_vdupq_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f16 (float16x4_t __a, float16x4_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfms_f16 (__a, __b, __aarch64_vdup_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f16 (float16x8_t __a, float16x8_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmsq_f16 (__a, __b, __aarch64_vdupq_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f16 (float16x4_t __a, float16x4_t __b, float16_t __c)
-{
- return vfms_f16 (__a, __b, vdup_n_f16 (__c));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f16 (float16x8_t __a, float16x8_t __b, float16_t __c)
-{
- return vfmsq_f16 (__a, __b, vdupq_n_f16 (__c));
-}
-
__extension__ extern __inline float16_t
__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
vmulxh_lane_f16 (float16_t __a, float16x4_t __b, const int __lane)
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 30bbd5878b6..d787f4f5f53 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -215,6 +215,13 @@ (define_mode_iterator VSFDF [V2SF V4SF V2DF DF SF])
;; Advanced SIMD single Float modes.
(define_mode_iterator VDQSF [V2SF V4SF])
+;; Advanced SIMD half and single Float modes.
+(define_mode_iterator VDQHSF [
+ (V4HF "TARGET_SIMD_F16INST")
+ (V8HF "TARGET_SIMD_F16INST")
+ V2SF V4SF
+])
+
;; Quad vector float modes with half/bfloat elements.
(define_mode_iterator VQ_BHF [V8HF V8BF])
diff --git a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
index c1aeb06e747..9571f6df727 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
@@ -19,16 +19,9 @@ test_vfma##q1##_lane##q2##_f##size (float##size##_t * res,
\
float##size##x##in1_lanes##_t a = vld1##q1##_f##size (res); \
float##size##x##in1_lanes##_t b = vld1##q1##_f##size (in1); \
float##size##x##in2_lanes##_t c; \
- if (in2_lanes > 1) \
- { \
- c = vld1##q2##_f##size (in2); \
- a = vfma##q1##_lane##q2##_f##size (a, b, c, 1); \
- } \
- else \
- { \
- c = vld1##q2##_f##size (in2 + 1);
\
- a = vfma##q1##_lane##q2##_f##size (a, b, c, 0); \
- } \
+ const int lane_no = in2_lanes > 1 ? 1 : 0; \
+ c = vld1##q2##_f##size (in2 + (in2_lanes == 1)); \
+ a = vfma##q1##_lane##q2##_f##size (a, b, c, lane_no);
\
vst1##q1##_f##size (res, a); \
}
@@ -99,20 +92,20 @@ main (int argc, char **argv)
/* vfma_laneq_f32.
vfma_lane_f32. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2s, v[0-9]+.2s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfmaq_lane_f32.
vfmaq_laneq_f32. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.4s, v[0-9]+.4s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfma_lane_f64.
vfma_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 1 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmadd d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmadd d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 2 { target aarch64_little_endian } } } */
/* vfmaq_lane_f64.
vfmaq_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 2 { target aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
index 3137ea91e80..0f7e1a76487 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
@@ -19,16 +19,9 @@ test_vfms##q1##_lane##q2##_f##size (float##size##_t * res,
\
float##size##x##in1_lanes##_t a = vld1##q1##_f##size (res); \
float##size##x##in1_lanes##_t b = vld1##q1##_f##size (in1); \
float##size##x##in2_lanes##_t c; \
- if (in2_lanes > 1) \
- { \
- c = vld1##q2##_f##size (in2); \
- a = vfms##q1##_lane##q2##_f##size (a, b, c, 1); \
- } \
- else \
- { \
- c = vld1##q2##_f##size (in2 + 1);
\
- a = vfms##q1##_lane##q2##_f##size (a, b, c, 0); \
- } \
+ const int lane_no = in2_lanes > 1 ? 1 : 0; \
+ c = vld1##q2##_f##size (in2 + (in2_lanes == 1)); \
+ a = vfms##q1##_lane##q2##_f##size (a, b, c, lane_no);
\
vst1##q1##_f##size (res, a); \
}
@@ -100,20 +93,20 @@ main (int argc, char **argv)
/* vfms_laneq_f32.
vfms_lane_f32. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2s, v[0-9]+.2s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfmsq_lane_f32.
vfmsq_laneq_f32. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.4s, v[0-9]+.4s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfms_lane_f64.
vfms_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 1 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmsub d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmsub d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 2 { target aarch64_little_endian } } } */
/* vfmsq_lane_f64.
vfmsq_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 2 { target aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
b/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
new file mode 100644
index 00000000000..7de3e90b867
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
@@ -0,0 +1,220 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+#define TEST_LANE(NAME, RET_TYPE, TYPE1, TYPE2, TYPE3)
\
+ RET_TYPE test_##NAME (TYPE1 a, TYPE2 b, TYPE3 c)
\
+ {
\
+ return NAME (a, b, c, VEC_LEN (c) - 1);
\
+ }
+
+/*
+** test_vfma_f16:
+** fmla v0\.4h, (v1\.4h, v2\.4h|v2\.4h, v1\.4h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f16, float16x4_t)
+
+/*
+** test_vfmaq_f16:
+** fmla v0\.8h, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f16, float16x8_t)
+
+/*
+** test_vfma_f32:
+** fmla v0\.2s, (v1\.2s, v2\.2s|v2\.2s, v1\.2s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f32, float32x2_t)
+
+/*
+** test_vfmaq_f32:
+** fmla v0\.4s, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f32, float32x4_t)
+
+/*
+** test_vfma_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f64, float64x1_t)
+
+/*
+** test_vfmaq_f64:
+** fmla v0\.2d, (v1\.2d, v2\.2d|v2\.2d, v1\.2d)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f64, float64x2_t)
+
+/*
+** test_vfma_n_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfma_n_f16, float16x4_t, float16x4_t, float16x4_t, float16_t)
+
+/*
+** test_vfmaq_n_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f16, float16x8_t, float16x8_t, float16x8_t, float16_t)
+
+/*
+** test_vfma_n_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfma_n_f32, float32x2_t, float32x2_t, float32x2_t, float32_t)
+
+/*
+** test_vfmaq_n_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f32, float32x4_t, float32x4_t, float32x4_t, float32_t)
+
+/*
+** test_vfma_n_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_TERNARY (vfma_n_f64, float64x1_t, float64x1_t, float64x1_t, float64_t)
+
+/*
+** test_vfmaq_n_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f64, float64x2_t, float64x2_t, float64x2_t, float64_t)
+
+/*
+** test_vfma_lane_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfma_lane_f16, float16x4_t, float16x4_t, float16x4_t, float16x4_t)
+
+/*
+** test_vfma_lane_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfma_lane_f32, float32x2_t, float32x2_t, float32x2_t, float32x2_t)
+
+/*
+** test_vfma_lane_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfma_lane_f64, float64x1_t, float64x1_t, float64x1_t, float64x1_t)
+
+/*
+** test_vfma_laneq_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f16, float16x4_t, float16x4_t, float16x4_t, float16x8_t)
+
+/*
+** test_vfma_laneq_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f32, float32x2_t, float32x2_t, float32x2_t, float32x4_t)
+
+/*
+** test_vfma_laneq_f64:
+** fmla d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f64, float64x1_t, float64x1_t, float64x1_t, float64x2_t)
+
+/*
+** test_vfmaq_lane_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f16, float16x8_t, float16x8_t, float16x8_t, float16x4_t)
+
+/*
+** test_vfmaq_lane_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f32, float32x4_t, float32x4_t, float32x4_t, float32x2_t)
+
+/*
+** test_vfmaq_lane_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f64, float64x2_t, float64x2_t, float64x2_t, float64x1_t)
+
+/*
+** test_vfmaq_laneq_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f16, float16x8_t, float16x8_t, float16x8_t, float16x8_t)
+
+/*
+** test_vfmaq_laneq_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f32, float32x4_t, float32x4_t, float32x4_t, float32x4_t)
+
+/*
+** test_vfmaq_laneq_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f64, float64x2_t, float64x2_t, float64x2_t, float64x2_t)
+
+/*
+** test_vfmah_lane_f16:
+** fmla h0, h1, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmah_lane_f16, float16_t, float16_t, float16_t, float16x4_t)
+
+/*
+** test_vfmah_laneq_f16:
+** fmla h0, h1, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmah_laneq_f16, float16_t, float16_t, float16_t, float16x8_t)
+
+/*
+** test_vfmas_lane_f32:
+** fmla s0, s1, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmas_lane_f32, float32_t, float32_t, float32_t, float32x2_t)
+
+/*
+** test_vfmas_laneq_f32:
+** fmla s0, s1, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmas_laneq_f32, float32_t, float32_t, float32_t, float32x4_t)
+
+/*
+** test_vfmad_lane_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfmad_lane_f64, float64_t, float64_t, float64_t, float64x1_t)
+
+/*
+** test_vfmad_laneq_f64:
+** fmla d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmad_laneq_f64, float64_t, float64_t, float64_t, float64x2_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
b/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
new file mode 100644
index 00000000000..c5bd09d3a61
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
@@ -0,0 +1,220 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+#define TEST_LANE(NAME, RET_TYPE, TYPE1, TYPE2, TYPE3)
\
+ RET_TYPE test_##NAME (TYPE1 a, TYPE2 b, TYPE3 c)
\
+ {
\
+ return NAME (a, b, c, VEC_LEN (c) - 1);
\
+ }
+
+/*
+** test_vfms_f16:
+** fmls v0\.4h, (v1\.4h, v2\.4h|v2\.4h, v1\.4h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f16, float16x4_t)
+
+/*
+** test_vfmsq_f16:
+** fmls v0\.8h, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f16, float16x8_t)
+
+/*
+** test_vfms_f32:
+** fmls v0\.2s, (v1\.2s, v2\.2s|v2\.2s, v1\.2s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f32, float32x2_t)
+
+/*
+** test_vfmsq_f32:
+** fmls v0\.4s, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f32, float32x4_t)
+
+/*
+** test_vfms_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f64, float64x1_t)
+
+/*
+** test_vfmsq_f64:
+** fmls v0\.2d, (v1\.2d, v2\.2d|v2\.2d, v1\.2d)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f64, float64x2_t)
+
+/*
+** test_vfms_n_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfms_n_f16, float16x4_t, float16x4_t, float16x4_t, float16_t)
+
+/*
+** test_vfmsq_n_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f16, float16x8_t, float16x8_t, float16x8_t, float16_t)
+
+/*
+** test_vfms_n_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfms_n_f32, float32x2_t, float32x2_t, float32x2_t, float32_t)
+
+/*
+** test_vfmsq_n_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f32, float32x4_t, float32x4_t, float32x4_t, float32_t)
+
+/*
+** test_vfms_n_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_TERNARY (vfms_n_f64, float64x1_t, float64x1_t, float64x1_t, float64_t)
+
+/*
+** test_vfmsq_n_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f64, float64x2_t, float64x2_t, float64x2_t, float64_t)
+
+/*
+** test_vfms_lane_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfms_lane_f16, float16x4_t, float16x4_t, float16x4_t, float16x4_t)
+
+/*
+** test_vfms_lane_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfms_lane_f32, float32x2_t, float32x2_t, float32x2_t, float32x2_t)
+
+/*
+** test_vfms_lane_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfms_lane_f64, float64x1_t, float64x1_t, float64x1_t, float64x1_t)
+
+/*
+** test_vfms_laneq_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f16, float16x4_t, float16x4_t, float16x4_t, float16x8_t)
+
+/*
+** test_vfms_laneq_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f32, float32x2_t, float32x2_t, float32x2_t, float32x4_t)
+
+/*
+** test_vfms_laneq_f64:
+** fmls d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f64, float64x1_t, float64x1_t, float64x1_t, float64x2_t)
+
+/*
+** test_vfmsq_lane_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f16, float16x8_t, float16x8_t, float16x8_t, float16x4_t)
+
+/*
+** test_vfmsq_lane_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f32, float32x4_t, float32x4_t, float32x4_t, float32x2_t)
+
+/*
+** test_vfmsq_lane_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f64, float64x2_t, float64x2_t, float64x2_t, float64x1_t)
+
+/*
+** test_vfmsq_laneq_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f16, float16x8_t, float16x8_t, float16x8_t, float16x8_t)
+
+/*
+** test_vfmsq_laneq_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f32, float32x4_t, float32x4_t, float32x4_t, float32x4_t)
+
+/*
+** test_vfmsq_laneq_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f64, float64x2_t, float64x2_t, float64x2_t, float64x2_t)
+
+/*
+** test_vfmsh_lane_f16:
+** fmls h0, h1, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmsh_lane_f16, float16_t, float16_t, float16_t, float16x4_t)
+
+/*
+** test_vfmsh_laneq_f16:
+** fmls h0, h1, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmsh_laneq_f16, float16_t, float16_t, float16_t, float16x8_t)
+
+/*
+** test_vfmss_lane_f32:
+** fmls s0, s1, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmss_lane_f32, float32_t, float32_t, float32_t, float32x2_t)
+
+/*
+** test_vfmss_laneq_f32:
+** fmls s0, s1, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmss_laneq_f32, float32_t, float32_t, float32_t, float32x4_t)
+
+/*
+** test_vfmsd_lane_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfmsd_lane_f64, float64_t, float64_t, float64_t, float64x1_t)
+
+/*
+** test_vfmsd_laneq_f64:
+** fmls d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmsd_laneq_f64, float64_t, float64_t, float64_t, float64x2_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
b/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
index c751924e8d1..cc2b3887947 100644
--- a/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
@@ -3,7 +3,7 @@
the top half of the vector register and no explicit zeroing instructions
are emitted. */
/* { dg-do compile } */
-/* { dg-options "-O" } */
+/* { dg-options "-O -fno-trapping-math" } */
#include <arm_neon.h>
@@ -65,4 +65,3 @@ OPTHREE (float32, 2, 4, f32, mla_lane, fma_lane, fms_lane)
/* { dg-final { scan-assembler-not {\tfmov\t} } } */
/* { dg-final { scan-assembler-not {\tmov\t} } } */
-
--
2.56.0
