gcc/ChangeLog:
* config/aarch64/aarch64-neon-builtins-base.cc
(build_ifn_call): New function.
(gimple_ifn::fold): Call `build_ifn_call`.
(struct gimple_fused_mla): New struct.
* config/aarch64/aarch64-neon-builtins-base.def
(vfma, vfmaq, vfma_n, vfmaq_n, vfma_lane, vfma_laneq,
vfmaq_lane, vfmaq_laneq, vfmas_lane, vfmas_laneq, vfmad_lane,
vfmad_laneq, vfms, vfmsq, vfms_n, vfmsq_n, vfms_lane,
vfms_laneq, vfmsq_lane, vfmsq_laneq, vfmss_lane, vfmss_laneq,
vfmsd_lane, vfmsd_laneq, and the FP16 forms of each): New
intrinsic function declarations.
* config/aarch64/aarch64-simd-builtins.def
(fma, fnma): Delete `__builtin_aarch64` function declarations.
* config/aarch64/aarch64-simd.md
(fma<mode>4<vczle><vczbe>,
*aarch64_fma4_elt<mode><vczle><vczbe>,
*aarch64_fma4_elt_<vswap_width_name><mode><vczle><vczbe>,
*aarch64_fma4_elt_from_dup<mode><vczle><vczbe>,
*aarch64_fma<mode>4_elt_to_scalar): Extend to cover HF vector
modes, reformat, add comments.
(fnma<mode>4<vczle><vczbe>,
*aarch64_fnma4_elt<mode><vczle><vczbe>
*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>
*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>
*aarch64_fnma<mode>4_elt_from_to_scalar):
Likewise.
(*aarch64_fma4_elt_to_64v2df, *aarch64_fnma4_elt_to_64v2df):
Rename to ...
(*aarch64_fma<mode>4_elt_to_scalar,
*aarch64_fnma<mode>4_elt_to_scalar): ...these.
(*aarch64_fnma4_elt<mode><vczle><vczbe>_alt,
*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>_alt,
*aarch64_fnma4_elt_from_dup<mode><cvzle><vczbe>_alt,
*aarch64_fnma4<mode>4_elt_to_scalar_alt): New insns to match
permuted variants.
* config/aarch64/arm_neon.h
(vfma_f64, vfma_f32, vfmaq_f32, vfmaq_f64, vfma_n_f32,
vfma_n_f64, vfmaq_n_f32, vfmaq_n_f64, vfma_lane_f32,
vfma_lane_f64, vfma_laneq_f32, vfma_laneq_f64,
vfmaq_lane_f32, vfmaq_lane_f64, vfmaq_laneq_f32,
vfmaq_laneq_f64, vfms_f64, vfms_f32, vfmsq_f32, vfmsq_f64,
vfms_n_f32, vfms_n_f64, vfmsq_n_f32, vfmsq_n_f64,
vfms_lane_f32, vfms_lane_f64, vfms_laneq_f32, vfms_laneq_f64,
vfmsq_lane_f32, vfmsq_lane_f64, vfmsq_laneq_f32,
vfmsq_laneq_f64, vfma_f16, vfmaq_f16, vfms_f16, vfmsq_f16,
vfma_lane_f16, vfmaq_lane_f16, vfma_laneq_f16,
vfmaq_laneq_f16, vfma_n_f16, vfmaq_n_f16, vfms_lane_f16,
vfmsq_lane_f16, vfms_laneq_f16, vfmsq_laneq_f16, vfms_n_f16,
vfmsq_n_f16): Delete function definitions.
* config/aarch64/iterators.md (VDQHSF): New mode iterator.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/fmla_intrinsic_1.c: Rewrite `TEST_VMLA`
so the lane choice is always valid and doesn't cause a compiler
error when trying to call `vfma`. Also remove excessive
backslashes from dejagnu directives.
* gcc.target/aarch64/fmls_intrinsic_1.c: Likewise.
* gcc.target/aarch64/simd/pr99195_3.c: Add
`-fno-trapping-math` to prevent the optimization barrier being
introduced and generating unnecessary `fmov`s.
* gcc.target/aarch64/neon/vfma.c: New test.
* gcc.target/aarch64/neon/vfms.c: New test.
---
.../aarch64/aarch64-neon-builtins-base.cc | 122 ++++-
.../aarch64/aarch64-neon-builtins-base.def | 54 ++
gcc/config/aarch64/aarch64-simd-builtins.def | 2 -
gcc/config/aarch64/aarch64-simd.md | 244 ++++++---
gcc/config/aarch64/arm_neon.h | 500 ------------------
gcc/config/aarch64/iterators.md | 7 +
.../gcc.target/aarch64/fmla_intrinsic_1.c | 25 +-
.../gcc.target/aarch64/fmls_intrinsic_1.c | 25 +-
gcc/testsuite/gcc.target/aarch64/neon/vfma.c | 220 ++++++++
gcc/testsuite/gcc.target/aarch64/neon/vfms.c | 220 ++++++++
.../gcc.target/aarch64/simd/pr99195_3.c | 3 +-
11 files changed, 780 insertions(+), 642 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vfma.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vfms.c
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
index bdb14273f5f..f96b3332dc4 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
@@ -176,6 +176,36 @@ one_element_vector_type_p (tree type)
return VECTOR_TYPE_P (type) && known_eq (TYPE_VECTOR_SUBPARTS (type), 1U);
}
+/* Build a call to the internal function IFN.
+ Handles singleton vector arguments specially: they are cast to their
+ inner element type before the function is called, and then the return value
+ is cast back to the expected singleton vector type. */
+tree
+build_ifn_call (gimple_folder &f, internal_fn ifn, tree ret_type,
+ vec<tree> &args)
+{
+ for (auto &arg : args)
+ if (one_element_vector_type_p (TREE_TYPE (arg)))
+ arg = f.force_val (build_cast (TREE_TYPE (TREE_TYPE (arg)), arg));
+
+ auto call = gimple_build_call_internal_vec (ifn, args);
+ gimple_seq_add_stmt_without_update (&f.stmts, call);
+
+ if (!one_element_vector_type_p (ret_type))
+ {
+ auto ret = is_gimple_reg_type (ret_type) ? make_ssa_name (ret_type)
+ : create_tmp_var (ret_type);
+ gimple_call_set_lhs (call, ret);
+ return ret;
+ }
+ else
+ {
+ auto ret = create_tmp_var (TREE_TYPE (ret_type));
+ gimple_call_set_lhs (call, ret);
+ return build_vec_dup (ret_type, ret);
+ }
+}
+
/* Base class for all function expanders.
At least one of `expand` or `fold` must be overriden by derived classes. */
class gimple_function_base : public function_base
@@ -644,44 +674,19 @@ public:
gimple *fold (gimple_folder &f) const override
{
tree lhs_type = TREE_TYPE (f.lhs);
- /* There are no instructions that handle V1DI or V1DF modes for these
- operations. They need to be specially handled by casting the vector to
- its element type and using it to feed the scalar version of the
- operation. The scalar result is then cast back to a vector and
returned.
- This matches the prior implementation that arm_neon.h used for these
- operations. */
- bool scalar_lhs = one_element_vector_type_p (lhs_type);
-
vec<tree> args{};
for (unsigned i = 0; i < gimple_call_num_args (f.call); i++)
{
tree arg = gimple_call_arg (f.call, i);
- /* Convert V1DI/V1DF arguments to corresponding scalar modes. */
- if (scalar_lhs && one_element_vector_type_p (TREE_TYPE (arg)))
- arg = f.force_val (build_cast (TREE_TYPE (lhs_type), arg));
/* .CLZ and .POPCOUNT take an unsigned argument. */
- else if ((m_ifn == IFN_CLZ || m_ifn == IFN_POPCOUNT)
- && !TYPE_UNSIGNED (TREE_TYPE (arg)))
+ if ((m_ifn == IFN_CLZ || m_ifn == IFN_POPCOUNT)
+ && !TYPE_UNSIGNED (TREE_TYPE (arg)))
arg = f.force_val (build_cast (unsigned_type_for (TREE_TYPE (arg)),
arg));
args.safe_push (arg);
}
-
- auto call = gimple_build_call_internal_vec (this->m_ifn, args);
- args.release ();
-
- if (!scalar_lhs)
- {
- gimple_call_set_lhs (call, f.lhs);
- return call;
- }
-
- /* Temporary scalar to hold the result. */
- tree res = create_tmp_var (TREE_TYPE (lhs_type));
- gimple_call_set_lhs (call, res);
- gimple_seq_add_stmt_without_update (&f.stmts, call);
- /* Build the actual vector that the call's result gets assigned to. */
- return gimple_build_assign (f.lhs, build_vec_dup (lhs_type, res));
+ return gimple_build_assign (f.lhs, build_ifn_call (f, this->m_ifn,
lhs_type,
+ args));
}
};
@@ -1019,6 +1024,46 @@ struct multiply_accumulate : public gimple_function_base
template <arith_kind kind> using gimple_mla = multiply_accumulate<kind, false>;
template <arith_kind kind> using gimple_mls = multiply_accumulate<kind, true>;
+template <arith_kind kind, bool negate>
+struct gimple_fused_mla : public gimple_function_base
+{
+ gimple *fold (gimple_folder &f) const override
+ {
+ tree acc = gimple_call_arg (f.call, 0);
+ tree lhs = gimple_call_arg (f.call, 1);
+ tree rhs = gimple_call_arg (f.call, 2);
+ tree lane_index;
+ tree ret_type = TREE_TYPE (f.lhs);
+
+ switch (kind)
+ {
+ case arith_kind::normal:
+ break;
+
+ case arith_kind::dup:
+ rhs = f.force_val (build_vec_dup (TREE_TYPE (lhs), rhs));
+ break;
+
+ case arith_kind::lane:
+ lane_index = gimple_call_arg (f.call, 3);
+ rhs = f.force_val (build_lane_get (rhs, lane_index));
+ if (VECTOR_TYPE_P (ret_type))
+ rhs = f.force_val (build_vec_dup (TREE_TYPE (lhs), rhs));
+ break;
+ }
+
+ vec<tree> args{};
+ args.safe_push (lhs);
+ args.safe_push (rhs);
+ args.safe_push (acc);
+ auto ret = build_ifn_call (f, negate ? IFN_FNMA : IFN_FMA, ret_type, args);
+ args.release ();
+ return this->maybe_optimization_barrier (f, ret);
+ }
+};
+template <arith_kind kind> using gimple_fma = gimple_fused_mla<kind, false>;
+template <arith_kind kind> using gimple_fms = gimple_fused_mla<kind, true>;
+
// Reinterpret
NEON_FUNCTION (vreinterpret, vreinterpretq, gimple_reinterpret,)
@@ -1060,6 +1105,25 @@ NEON_FUNCTION (vmls_n, vmlsq_n,
gimple_mls<arith_kind::dup>,)
NEON_FUNCTION (vmls_lane, vmls_laneq, vmlsq_lane, vmlsq_laneq,
gimple_mls<arith_kind::lane>,)
+// Multiply-accumulate (fused)
+NEON_FUNCTION (vfma, vfmaq, gimple_fma<arith_kind::normal>,);
+NEON_FUNCTION (vfma_n, vfmaq_n, gimple_fma<arith_kind::dup>,);
+NEON_FUNCTION (vfmah_lane, vfmah_laneq,
+ vfmas_lane, vfmas_laneq,
+ vfmad_lane, vfmad_laneq,
+ vfma_lane, vfma_laneq,
+ vfmaq_lane, vfmaq_laneq,
+ gimple_fma<arith_kind::lane>,);
+
+NEON_FUNCTION (vfms, vfmsq, gimple_fms<arith_kind::normal>,);
+NEON_FUNCTION (vfms_n, vfmsq_n, gimple_fms<arith_kind::dup>,);
+NEON_FUNCTION (vfmsh_lane, vfmsh_laneq,
+ vfmss_lane, vfmss_laneq,
+ vfmsd_lane, vfmsd_laneq,
+ vfms_lane, vfms_laneq,
+ vfmsq_lane, vfmsq_laneq,
+ gimple_fms<arith_kind::lane>,);
+
// Saturating arithmetic
NEON_FUNCTION (vqaddb, vqaddh, vqadds, vqaddd, vqadd, vqaddq, gimple_ifn,
(IFN_SAT_ADD))
NEON_FUNCTION (vqsubb, vqsubh, vqsubs, vqsubd, vqsub, vqsubq, gimple_ifn,
(IFN_SAT_SUB))
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.def
b/gcc/config/aarch64/aarch64-neon-builtins-base.def
index 40aa1a656d7..e519e47beb2 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.def
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.def
@@ -296,6 +296,60 @@ DEF_NEON_FUNCTION (vmlsq_lane, neon_mla_lane,
("Q0,Q0,Q0,D0,ss32", lane<3>))
DEF_NEON_FUNCTION (vmlsq_laneq, neon_mla_lane, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
#undef REQUIRED_EXTENSIONS
+// Fused multiply-accumulate
+#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
+DEF_NEON_FUNCTION (vfma, sd_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmaq, sd_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfma_n, sd_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmaq_n, sd_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfma_lane, sd_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfma_laneq, sd_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_lane, sd_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_laneq, sd_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmas_lane, s_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmas_laneq, s_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmad_lane, d_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmad_laneq, d_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+
+DEF_NEON_FUNCTION (vfms, sd_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmsq, sd_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfms_n, sd_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmsq_n, sd_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfms_lane, sd_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfms_laneq, sd_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_lane, sd_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_laneq, sd_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmss_lane, s_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmss_laneq, s_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsd_lane, d_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsd_laneq, d_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+#undef REQUIRED_EXTENSIONS
+
+// Fused multiply-accumulate (FP16)
+#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD |
AARCH64_FL_F16)
+DEF_NEON_FUNCTION (vfma, h_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmaq, h_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfma_n, h_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmaq_n, h_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfma_lane, h_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfma_laneq, h_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_lane, h_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_laneq, h_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmah_lane, h_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmah_laneq, h_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+
+DEF_NEON_FUNCTION (vfms, h_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmsq, h_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfms_n, h_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmsq_n, h_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfms_lane, h_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfms_laneq, h_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_lane, h_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_laneq, h_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsh_lane, h_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsh_laneq, h_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+#undef REQUIRED_EXTENSIONS
+
// Bitwise operations
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
DEF_NEON_FUNCTION (vand, all_integer, ("D0,D0,D0"))
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def
b/gcc/config/aarch64/aarch64-simd-builtins.def
index 1fb7c763990..3792ec3edf4 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -669,10 +669,8 @@
BUILTIN_VALLP_NO_DI (STORE1_P, st1, 0, STORE)
/* Implemented by fma<mode>4. */
- BUILTIN_VHSDF (TERNOP, fma, 4, FP)
VAR1 (TERNOP, fma, 4, FP, hf)
/* Implemented by fnma<mode>4. */
- BUILTIN_VHSDF (TERNOP, fnma, 4, FP)
VAR1 (TERNOP, fnma, 4, FP, hf)
/* Implemented by aarch64_crypto_pmull<mode>. */
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index a42e1a60c58..7cc0d97b7c3 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -2962,9 +2962,9 @@ (define_expand "aarch64_<su>mlsl_hi_laneq<mode>"
)
;; FP vector operations.
-;; AArch64 AdvSIMD supports single-precision (32-bit) and
+;; AArch64 AdvSIMD supports single-precision (32-bit) and
;; double-precision (64-bit) floating-point data types and arithmetic as
-;; defined by the IEEE 754-2008 standard. This makes them vectorizable
+;; defined by the IEEE 754-2008 standard. This makes them vectorizable
;; without the need for -ffast-math or -funsafe-math-optimizations.
;;
;; Floating-point operations can raise an exception. Vectorizing such
@@ -3059,78 +3059,95 @@ (define_insn "abs<mode>2<vczle><vczbe>"
[(set_attr "type" "neon_fp_abs_<stype><q>")]
)
+;; vfma_f16, vfmaq_f16
+;; vfma_f32, vfmaq_f32
+;; vfmaq_f64
(define_insn "fma<mode>4<vczle><vczbe>"
[(set (match_operand:VHSDF 0 "register_operand" "=w")
- (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
- (match_operand:VHSDF 2 "register_operand" "w")
- (match_operand:VHSDF 3 "register_operand" "0")))]
+ (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
+ (match_operand:VHSDF 2 "register_operand" "w")
+ (match_operand:VHSDF 3 "register_operand" "0")))]
"TARGET_SIMD"
"fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
[(set_attr "type" "neon_fp_mla_<stype><q>")]
)
+;; vfma_lane_f16, vfmaq_laneq_f16
+;; vfma_lane_f32, vfmaq_laneq_f32
+;; vfmaq_laneq_f64
(define_insn "*aarch64_fma4_elt<mode><vczle><vczbe>"
- [(set (match_operand:VDQF 0 "register_operand" "=w")
- (fma:VDQF
- (vec_duplicate:VDQF
- (vec_select:<VEL>
- (match_operand:VDQF 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQF 3 "register_operand" "w")
- (match_operand:VDQF 4 "register_operand" "0")))]
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (vec_duplicate:VHSDF
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VHSDF 3 "register_operand" "w")
+ (match_operand:VHSDF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfma_laneq_f16, vfmaq_lane_f16
+;; vfma_laneq_f32, vfmaq_lane_f32
(define_insn "*aarch64_fma4_elt_<vswap_width_name><mode><vczle><vczbe>"
- [(set (match_operand:VDQSF 0 "register_operand" "=w")
- (fma:VDQSF
- (vec_duplicate:VDQSF
- (vec_select:<VEL>
- (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQSF 3 "register_operand" "w")
- (match_operand:VDQSF 4 "register_operand" "0")))]
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (vec_duplicate:VDQHSF
+ (vec_select:<VEL>
+ (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VDQHSF 3 "register_operand" "w")
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfma_n_f16, vfmaq_n_f16
+;; vfma_n_f32, vfmaq_n_f64
+;; vfmaq_n_f64 vfmaq_lane_f64
(define_insn "*aarch64_fma4_elt_from_dup<mode><vczle><vczbe>"
[(set (match_operand:VMUL 0 "register_operand" "=w")
- (fma:VMUL
- (vec_duplicate:VMUL
- (match_operand:<VEL> 1 "register_operand" "<h_con>"))
- (match_operand:VMUL 2 "register_operand" "w")
- (match_operand:VMUL 3 "register_operand" "0")))]
+ (fma:VMUL
+ (vec_duplicate:VMUL
+ (match_operand:<VEL> 1 "register_operand" "<h_con>"))
+ (match_operand:VMUL 2 "register_operand" "w")
+ (match_operand:VMUL 3 "register_operand" "0")))]
"TARGET_SIMD"
"fmla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
[(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
)
-(define_insn "*aarch64_fma4_elt_to_64v2df"
- [(set (match_operand:DF 0 "register_operand" "=w")
- (fma:DF
- (vec_select:DF
- (match_operand:V2DF 1 "register_operand" "w")
- (parallel [(match_operand:SI 2 "immediate_operand")]))
- (match_operand:DF 3 "register_operand" "w")
- (match_operand:DF 4 "register_operand" "0")))]
+;; vfmah_lane_f16, vfmah_laneq_f16
+;; vfmas_lane_f32, vfmas_laneq_f32
+;; vfma_laneq_f64, vfmad_laneq_f64
+(define_insn "*aarch64_fma<mode>4_elt_to_scalar"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))
+ (match_operand:<VEL> 3 "register_operand" "w")
+ (match_operand:<VEL> 4 "register_operand" "0")))]
"TARGET_SIMD"
{
- operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
- return "fmla\\t%0.2d, %3.2d, %1.d[%2]";
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmla\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_d_scalar_q")]
+ [(set_attr "type" "neon_fp_mla_<stype>_q")]
)
+;; vfms_f16, vfmsq_f16
+;; vfms_f32, vfmsq_f32
+;; vfmsq_f64
(define_insn "fnma<mode>4<vczle><vczbe>"
[(set (match_operand:VHSDF 0 "register_operand" "=w")
(fma:VHSDF
@@ -3142,70 +3159,143 @@ (define_insn "fnma<mode>4<vczle><vczbe>"
[(set_attr "type" "neon_fp_mla_<stype><q>")]
)
+;; vfms_lane_f16, vfmsq_laneq_f16,
+;; vfms_lane_f32, vfmsq_laneq_f32
+;; vfmsq_laneq_f64
(define_insn "*aarch64_fnma4_elt<mode><vczle><vczbe>"
- [(set (match_operand:VDQF 0 "register_operand" "=w")
- (fma:VDQF
- (neg:VDQF
- (match_operand:VDQF 3 "register_operand" "w"))
- (vec_duplicate:VDQF
- (vec_select:<VEL>
- (match_operand:VDQF 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQF 4 "register_operand" "0")))]
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (neg:VHSDF
+ (vec_duplicate:VHSDF
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))))
+ (match_operand:VHSDF 3 "register_operand" "w")
+ (match_operand:VHSDF 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ {
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
+ }
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
+)
+(define_insn "*aarch64_fnma4_elt<mode><vczle><vczbe>_alt"
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (neg:VHSDF
+ (match_operand:VHSDF 3 "register_operand" "w"))
+ (vec_duplicate:VHSDF
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VHSDF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfms_laneq_f16, vfmsq_lane_f16
+;; vfms_laneq_f32, vfmsq_lane_f32
(define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>"
- [(set (match_operand:VDQSF 0 "register_operand" "=w")
- (fma:VDQSF
- (neg:VDQSF
- (match_operand:VDQSF 3 "register_operand" "w"))
- (vec_duplicate:VDQSF
- (vec_select:<VEL>
- (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQSF 4 "register_operand" "0")))]
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (neg:VDQHSF
+ (vec_duplicate:VDQHSF
+ (vec_select:<VEL>
+ (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))))
+ (match_operand:VDQHSF 3 "register_operand" "w")
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ {
+ operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
+ return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
+ }
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
+)
+(define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>_alt"
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (neg:VDQHSF
+ (match_operand:VDQHSF 3 "register_operand" "w"))
+ (vec_duplicate:VDQHSF
+ (vec_select:<VEL>
+ (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
+ [(set_attr "type" "neon_fp_mla_<stype>_scalar<q>")]
)
+;; vfms_n_f16, vfmsq_n_f16
+;; vfms_n_f32, vfmsq_n_f32
+;; vfmsq_n_f64, vfmq_lane_f64
(define_insn "*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>"
[(set (match_operand:VMUL 0 "register_operand" "=w")
- (fma:VMUL
- (neg:VMUL
- (match_operand:VMUL 2 "register_operand" "w"))
- (vec_duplicate:VMUL
- (match_operand:<VEL> 1 "register_operand" "<h_con>"))
- (match_operand:VMUL 3 "register_operand" "0")))]
+ (fma:VMUL
+ (neg:VMUL
+ (vec_duplicate:VMUL
+ (match_operand:<VEL> 1 "register_operand" "<h_con>")))
+ (match_operand:VMUL 2 "register_operand" "w")
+ (match_operand:VMUL 3 "register_operand" "0")))]
+ "TARGET_SIMD"
+ "fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
+ [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
+)
+(define_insn "*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>_alt"
+ [(set (match_operand:VMUL 0 "register_operand" "=w")
+ (fma:VMUL
+ (neg:VMUL
+ (match_operand:VMUL 2 "register_operand" "w"))
+ (vec_duplicate:VMUL
+ (match_operand:<VEL> 1 "register_operand" "<h_con>"))
+ (match_operand:VMUL 3 "register_operand" "0")))]
"TARGET_SIMD"
"fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
[(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
)
-(define_insn "*aarch64_fnma4_elt_to_64v2df"
- [(set (match_operand:DF 0 "register_operand" "=w")
- (fma:DF
- (vec_select:DF
- (match_operand:V2DF 1 "register_operand" "w")
- (parallel [(match_operand:SI 2 "immediate_operand")]))
- (neg:DF
- (match_operand:DF 3 "register_operand" "w"))
- (match_operand:DF 4 "register_operand" "0")))]
+;; vfmsh_lane_f16, vfmsh_laneq_f16
+;; vfmss_lane_f32, vfmss_laneq_f32
+;; vfms_laneq_f64, vfmsd_laneq_f64
+(define_insn "*aarch64_fnma<mode>4_elt_to_scalar"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (neg:<VEL>
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:<VEL> 3 "register_operand" "w")
+ (match_operand:<VEL> 4 "register_operand" "0")))]
"TARGET_SIMD"
{
- operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
- return "fmls\\t%0.2d, %3.2d, %1.d[%2]";
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmls\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]";
+ }
+ [(set_attr "type" "neon_fp_mla_<stype>_q")]
+)
+(define_insn "*aarch64_fnma<mode>4_elt_to_scalar_alt"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (neg:<VEL>
+ (match_operand:<VEL> 3 "register_operand" "w"))
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))
+ (match_operand:<VEL> 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ {
+ operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+ return "fmls\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]";
}
- [(set_attr "type" "neon_fp_mla_d_scalar_q")]
+ [(set_attr "type" "neon_fp_mla_<stype>_q")]
)
;; Vector versions of the floating-point frint patterns.
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index 71b93a6adf5..db9d5c4c185 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -6858,352 +6858,6 @@ vcvtpq_u64_f64 (float64x2_t __a)
return __builtin_aarch64_lceiluv2dfv2di_us (__a);
}
-/* vfma */
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f64 (float64x1_t __a, float64x1_t __b, float64x1_t __c)
-{
- return (float64x1_t) {__builtin_fma (__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f32 (float32x2_t __a, float32x2_t __b, float32x2_t __c)
-{
- return __builtin_aarch64_fmav2sf (__b, __c, __a);
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f32 (float32x4_t __a, float32x4_t __b, float32x4_t __c)
-{
- return __builtin_aarch64_fmav4sf (__b, __c, __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f64 (float64x2_t __a, float64x2_t __b, float64x2_t __c)
-{
- return __builtin_aarch64_fmav2df (__b, __c, __a);
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f32 (float32x2_t __a, float32x2_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav2sf (__b, vdup_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f64 (float64x1_t __a, float64x1_t __b, float64_t __c)
-{
- return (float64x1_t) {__b[0] * __c + __a[0]};
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f32 (float32x4_t __a, float32x4_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav4sf (__b, vdupq_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f64 (float64x2_t __a, float64x2_t __b, float64_t __c)
-{
- return __builtin_aarch64_fmav2df (__b, vdupq_n_f64 (__c), __a);
-}
-
-/* vfma_lane */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f32 (float32x2_t __a, float32x2_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (__b,
- __aarch64_vdup_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f64 (float64x1_t __a, float64x1_t __b,
- float64x1_t __c, const int __lane)
-{
- return (float64x1_t) {__builtin_fma (__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmad_lane_f64 (float64_t __a, float64_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_fma (__b, __c[0], __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmas_lane_f32 (float32_t __a, float32_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_fmaf (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfma_laneq */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f32 (float32x2_t __a, float32x2_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (__b,
- __aarch64_vdup_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f64 (float64x1_t __a, float64x1_t __b,
- float64x2_t __c, const int __lane)
-{
- float64_t __c0 = __aarch64_vget_lane_any (__c, __lane);
- return (float64x1_t) {__builtin_fma (__b[0], __c0, __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmad_laneq_f64 (float64_t __a, float64_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_fma (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmas_laneq_f32 (float32_t __a, float32_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_fmaf (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfmaq_lane */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f32 (float32x4_t __a, float32x4_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (__b,
- __aarch64_vdupq_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f64 (float64x2_t __a, float64x2_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (__b, vdupq_n_f64 (__c[0]), __a);
-}
-
-/* vfmaq_laneq */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f32 (float32x4_t __a, float32x4_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (__b,
- __aarch64_vdupq_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f64 (float64x2_t __a, float64x2_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (__b,
- __aarch64_vdupq_laneq_f64 (__c, __lane),
- __a);
-}
-
-/* vfms */
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f64 (float64x1_t __a, float64x1_t __b, float64x1_t __c)
-{
- return (float64x1_t) {__builtin_fma (-__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f32 (float32x2_t __a, float32x2_t __b, float32x2_t __c)
-{
- return __builtin_aarch64_fmav2sf (-__b, __c, __a);
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f32 (float32x4_t __a, float32x4_t __b, float32x4_t __c)
-{
- return __builtin_aarch64_fmav4sf (-__b, __c, __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f64 (float64x2_t __a, float64x2_t __b, float64x2_t __c)
-{
- return __builtin_aarch64_fmav2df (-__b, __c, __a);
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f32 (float32x2_t __a, float32x2_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav2sf (-__b, vdup_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f64 (float64x1_t __a, float64x1_t __b, float64_t __c)
-{
- return (float64x1_t) {-__b[0] * __c + __a[0]};
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f32 (float32x4_t __a, float32x4_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav4sf (-__b, vdupq_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f64 (float64x2_t __a, float64x2_t __b, float64_t __c)
-{
- return __builtin_aarch64_fmav2df (-__b, vdupq_n_f64 (__c), __a);
-}
-
-/* vfms_lane */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f32 (float32x2_t __a, float32x2_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (-__b,
- __aarch64_vdup_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f64 (float64x1_t __a, float64x1_t __b,
- float64x1_t __c, const int __lane)
-{
- return (float64x1_t) {__builtin_fma (-__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsd_lane_f64 (float64_t __a, float64_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_fma (-__b, __c[0], __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmss_lane_f32 (float32_t __a, float32_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_fmaf (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfms_laneq */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f32 (float32x2_t __a, float32x2_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (-__b,
- __aarch64_vdup_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f64 (float64x1_t __a, float64x1_t __b,
- float64x2_t __c, const int __lane)
-{
- float64_t __c0 = __aarch64_vget_lane_any (__c, __lane);
- return (float64x1_t) {__builtin_fma (-__b[0], __c0, __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsd_laneq_f64 (float64_t __a, float64_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_fma (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmss_laneq_f32 (float32_t __a, float32_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_fmaf (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfmsq_lane */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f32 (float32x4_t __a, float32x4_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (-__b,
- __aarch64_vdupq_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f64 (float64x2_t __a, float64x2_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (-__b, vdupq_n_f64 (__c[0]), __a);
-}
-
-/* vfmsq_laneq */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f32 (float32x4_t __a, float32x4_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (-__b,
- __aarch64_vdupq_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f64 (float64x2_t __a, float64x2_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (-__b,
- __aarch64_vdupq_laneq_f64 (__c, __lane),
- __a);
-}
-
/* vld1 */
__extension__ extern __inline float16x4_t
@@ -17260,162 +16914,8 @@ vrsqrtsq_f16 (float16x8_t __a, float16x8_t __b)
return __builtin_aarch64_rsqrtsv8hf (__a, __b);
}
-/* ARMv8.2-A FP16 three operands vector intrinsics. */
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f16 (float16x4_t __a, float16x4_t __b, float16x4_t __c)
-{
- return __builtin_aarch64_fmav4hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f16 (float16x8_t __a, float16x8_t __b, float16x8_t __c)
-{
- return __builtin_aarch64_fmav8hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f16 (float16x4_t __a, float16x4_t __b, float16x4_t __c)
-{
- return __builtin_aarch64_fnmav4hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f16 (float16x8_t __a, float16x8_t __b, float16x8_t __c)
-{
- return __builtin_aarch64_fnmav8hf (__b, __c, __a);
-}
-
/* ARMv8.2-A FP16 lane vector intrinsics. */
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmah_lane_f16 (float16_t __a, float16_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmah_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmah_laneq_f16 (float16_t __a, float16_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmah_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f16 (float16x4_t __a, float16x4_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfma_f16 (__a, __b, __aarch64_vdup_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f16 (float16x8_t __a, float16x8_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmaq_f16 (__a, __b, __aarch64_vdupq_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f16 (float16x4_t __a, float16x4_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfma_f16 (__a, __b, __aarch64_vdup_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f16 (float16x8_t __a, float16x8_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmaq_f16 (__a, __b, __aarch64_vdupq_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f16 (float16x4_t __a, float16x4_t __b, float16_t __c)
-{
- return vfma_f16 (__a, __b, vdup_n_f16 (__c));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f16 (float16x8_t __a, float16x8_t __b, float16_t __c)
-{
- return vfmaq_f16 (__a, __b, vdupq_n_f16 (__c));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsh_lane_f16 (float16_t __a, float16_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmsh_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsh_laneq_f16 (float16_t __a, float16_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmsh_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f16 (float16x4_t __a, float16x4_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfms_f16 (__a, __b, __aarch64_vdup_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f16 (float16x8_t __a, float16x8_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmsq_f16 (__a, __b, __aarch64_vdupq_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f16 (float16x4_t __a, float16x4_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfms_f16 (__a, __b, __aarch64_vdup_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f16 (float16x8_t __a, float16x8_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmsq_f16 (__a, __b, __aarch64_vdupq_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f16 (float16x4_t __a, float16x4_t __b, float16_t __c)
-{
- return vfms_f16 (__a, __b, vdup_n_f16 (__c));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f16 (float16x8_t __a, float16x8_t __b, float16_t __c)
-{
- return vfmsq_f16 (__a, __b, vdupq_n_f16 (__c));
-}
-
__extension__ extern __inline float16_t
__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
vmulxh_lane_f16 (float16_t __a, float16x4_t __b, const int __lane)
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 30bbd5878b6..d787f4f5f53 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -215,6 +215,13 @@ (define_mode_iterator VSFDF [V2SF V4SF V2DF DF SF])
;; Advanced SIMD single Float modes.
(define_mode_iterator VDQSF [V2SF V4SF])
+;; Advanced SIMD half and single Float modes.
+(define_mode_iterator VDQHSF [
+ (V4HF "TARGET_SIMD_F16INST")
+ (V8HF "TARGET_SIMD_F16INST")
+ V2SF V4SF
+])
+
;; Quad vector float modes with half/bfloat elements.
(define_mode_iterator VQ_BHF [V8HF V8BF])
diff --git a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
index c1aeb06e747..9571f6df727 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
@@ -19,16 +19,9 @@ test_vfma##q1##_lane##q2##_f##size (float##size##_t * res, \
float##size##x##in1_lanes##_t a = vld1##q1##_f##size (res); \
float##size##x##in1_lanes##_t b = vld1##q1##_f##size (in1); \
float##size##x##in2_lanes##_t c; \
- if (in2_lanes > 1) \
- { \
- c = vld1##q2##_f##size (in2); \
- a = vfma##q1##_lane##q2##_f##size (a, b, c, 1); \
- } \
- else \
- { \
- c = vld1##q2##_f##size (in2 + 1); \
- a = vfma##q1##_lane##q2##_f##size (a, b, c, 0); \
- } \
+ const int lane_no = in2_lanes > 1 ? 1 : 0; \
+ c = vld1##q2##_f##size (in2 + (in2_lanes == 1)); \
+ a = vfma##q1##_lane##q2##_f##size (a, b, c, lane_no); \
vst1##q1##_f##size (res, a); \
}
@@ -99,20 +92,20 @@ main (int argc, char **argv)
/* vfma_laneq_f32.
vfma_lane_f32. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2s, v[0-9]+.2s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfmaq_lane_f32.
vfmaq_laneq_f32. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.4s, v[0-9]+.4s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfma_lane_f64.
vfma_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+" 1 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmadd d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmadd d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 2 { target aarch64_little_endian } } } */
/* vfmaq_lane_f64.
vfmaq_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 2 { target aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
index 3137ea91e80..0f7e1a76487 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
@@ -19,16 +19,9 @@ test_vfms##q1##_lane##q2##_f##size (float##size##_t * res, \
float##size##x##in1_lanes##_t a = vld1##q1##_f##size (res); \
float##size##x##in1_lanes##_t b = vld1##q1##_f##size (in1); \
float##size##x##in2_lanes##_t c; \
- if (in2_lanes > 1) \
- { \
- c = vld1##q2##_f##size (in2); \
- a = vfms##q1##_lane##q2##_f##size (a, b, c, 1); \
- } \
- else \
- { \
- c = vld1##q2##_f##size (in2 + 1); \
- a = vfms##q1##_lane##q2##_f##size (a, b, c, 0); \
- } \
+ const int lane_no = in2_lanes > 1 ? 1 : 0; \
+ c = vld1##q2##_f##size (in2 + (in2_lanes == 1)); \
+ a = vfms##q1##_lane##q2##_f##size (a, b, c, lane_no); \
vst1##q1##_f##size (res, a); \
}
@@ -100,20 +93,20 @@ main (int argc, char **argv)
/* vfms_laneq_f32.
vfms_lane_f32. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2s, v[0-9]+.2s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfmsq_lane_f32.
vfmsq_laneq_f32. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.4s, v[0-9]+.4s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfms_lane_f64.
vfms_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+" 1 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmsub d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmsub d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 2 { target aarch64_little_endian } } } */
/* vfmsq_lane_f64.
vfmsq_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 2 { target aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
b/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
new file mode 100644
index 00000000000..7de3e90b867
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
@@ -0,0 +1,220 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+#define TEST_LANE(NAME, RET_TYPE, TYPE1, TYPE2, TYPE3)
\
+ RET_TYPE test_##NAME (TYPE1 a, TYPE2 b, TYPE3 c)
\
+ {
\
+ return NAME (a, b, c, VEC_LEN (c) - 1);
\
+ }
+
+/*
+** test_vfma_f16:
+** fmla v0\.4h, (v1\.4h, v2\.4h|v2\.4h, v1\.4h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f16, float16x4_t)
+
+/*
+** test_vfmaq_f16:
+** fmla v0\.8h, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f16, float16x8_t)
+
+/*
+** test_vfma_f32:
+** fmla v0\.2s, (v1\.2s, v2\.2s|v2\.2s, v1\.2s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f32, float32x2_t)
+
+/*
+** test_vfmaq_f32:
+** fmla v0\.4s, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f32, float32x4_t)
+
+/*
+** test_vfma_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f64, float64x1_t)
+
+/*
+** test_vfmaq_f64:
+** fmla v0\.2d, (v1\.2d, v2\.2d|v2\.2d, v1\.2d)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f64, float64x2_t)
+
+/*
+** test_vfma_n_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfma_n_f16, float16x4_t, float16x4_t, float16x4_t, float16_t)
+
+/*
+** test_vfmaq_n_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f16, float16x8_t, float16x8_t, float16x8_t, float16_t)
+
+/*
+** test_vfma_n_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfma_n_f32, float32x2_t, float32x2_t, float32x2_t, float32_t)
+
+/*
+** test_vfmaq_n_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f32, float32x4_t, float32x4_t, float32x4_t, float32_t)
+
+/*
+** test_vfma_n_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_TERNARY (vfma_n_f64, float64x1_t, float64x1_t, float64x1_t, float64_t)
+
+/*
+** test_vfmaq_n_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f64, float64x2_t, float64x2_t, float64x2_t, float64_t)
+
+/*
+** test_vfma_lane_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfma_lane_f16, float16x4_t, float16x4_t, float16x4_t, float16x4_t)
+
+/*
+** test_vfma_lane_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfma_lane_f32, float32x2_t, float32x2_t, float32x2_t, float32x2_t)
+
+/*
+** test_vfma_lane_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfma_lane_f64, float64x1_t, float64x1_t, float64x1_t, float64x1_t)
+
+/*
+** test_vfma_laneq_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f16, float16x4_t, float16x4_t, float16x4_t, float16x8_t)
+
+/*
+** test_vfma_laneq_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f32, float32x2_t, float32x2_t, float32x2_t, float32x4_t)
+
+/*
+** test_vfma_laneq_f64:
+** fmla d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f64, float64x1_t, float64x1_t, float64x1_t, float64x2_t)
+
+/*
+** test_vfmaq_lane_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f16, float16x8_t, float16x8_t, float16x8_t, float16x4_t)
+
+/*
+** test_vfmaq_lane_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f32, float32x4_t, float32x4_t, float32x4_t, float32x2_t)
+
+/*
+** test_vfmaq_lane_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f64, float64x2_t, float64x2_t, float64x2_t, float64x1_t)
+
+/*
+** test_vfmaq_laneq_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f16, float16x8_t, float16x8_t, float16x8_t, float16x8_t)
+
+/*
+** test_vfmaq_laneq_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f32, float32x4_t, float32x4_t, float32x4_t, float32x4_t)
+
+/*
+** test_vfmaq_laneq_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f64, float64x2_t, float64x2_t, float64x2_t, float64x2_t)
+
+/*
+** test_vfmah_lane_f16:
+** fmla h0, h1, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmah_lane_f16, float16_t, float16_t, float16_t, float16x4_t)
+
+/*
+** test_vfmah_laneq_f16:
+** fmla h0, h1, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmah_laneq_f16, float16_t, float16_t, float16_t, float16x8_t)
+
+/*
+** test_vfmas_lane_f32:
+** fmla s0, s1, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmas_lane_f32, float32_t, float32_t, float32_t, float32x2_t)
+
+/*
+** test_vfmas_laneq_f32:
+** fmla s0, s1, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmas_laneq_f32, float32_t, float32_t, float32_t, float32x4_t)
+
+/*
+** test_vfmad_lane_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfmad_lane_f64, float64_t, float64_t, float64_t, float64x1_t)
+
+/*
+** test_vfmad_laneq_f64:
+** fmla d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmad_laneq_f64, float64_t, float64_t, float64_t, float64x2_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
b/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
new file mode 100644
index 00000000000..c5bd09d3a61
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
@@ -0,0 +1,220 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+#define TEST_LANE(NAME, RET_TYPE, TYPE1, TYPE2, TYPE3)
\
+ RET_TYPE test_##NAME (TYPE1 a, TYPE2 b, TYPE3 c)
\
+ {
\
+ return NAME (a, b, c, VEC_LEN (c) - 1);
\
+ }
+
+/*
+** test_vfms_f16:
+** fmls v0\.4h, (v1\.4h, v2\.4h|v2\.4h, v1\.4h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f16, float16x4_t)
+
+/*
+** test_vfmsq_f16:
+** fmls v0\.8h, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f16, float16x8_t)
+
+/*
+** test_vfms_f32:
+** fmls v0\.2s, (v1\.2s, v2\.2s|v2\.2s, v1\.2s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f32, float32x2_t)
+
+/*
+** test_vfmsq_f32:
+** fmls v0\.4s, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f32, float32x4_t)
+
+/*
+** test_vfms_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f64, float64x1_t)
+
+/*
+** test_vfmsq_f64:
+** fmls v0\.2d, (v1\.2d, v2\.2d|v2\.2d, v1\.2d)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f64, float64x2_t)
+
+/*
+** test_vfms_n_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfms_n_f16, float16x4_t, float16x4_t, float16x4_t, float16_t)
+
+/*
+** test_vfmsq_n_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f16, float16x8_t, float16x8_t, float16x8_t, float16_t)
+
+/*
+** test_vfms_n_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfms_n_f32, float32x2_t, float32x2_t, float32x2_t, float32_t)
+
+/*
+** test_vfmsq_n_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f32, float32x4_t, float32x4_t, float32x4_t, float32_t)
+
+/*
+** test_vfms_n_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_TERNARY (vfms_n_f64, float64x1_t, float64x1_t, float64x1_t, float64_t)
+
+/*
+** test_vfmsq_n_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f64, float64x2_t, float64x2_t, float64x2_t, float64_t)
+
+/*
+** test_vfms_lane_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfms_lane_f16, float16x4_t, float16x4_t, float16x4_t, float16x4_t)
+
+/*
+** test_vfms_lane_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfms_lane_f32, float32x2_t, float32x2_t, float32x2_t, float32x2_t)
+
+/*
+** test_vfms_lane_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfms_lane_f64, float64x1_t, float64x1_t, float64x1_t, float64x1_t)
+
+/*
+** test_vfms_laneq_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f16, float16x4_t, float16x4_t, float16x4_t, float16x8_t)
+
+/*
+** test_vfms_laneq_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f32, float32x2_t, float32x2_t, float32x2_t, float32x4_t)
+
+/*
+** test_vfms_laneq_f64:
+** fmls d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f64, float64x1_t, float64x1_t, float64x1_t, float64x2_t)
+
+/*
+** test_vfmsq_lane_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f16, float16x8_t, float16x8_t, float16x8_t, float16x4_t)
+
+/*
+** test_vfmsq_lane_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f32, float32x4_t, float32x4_t, float32x4_t, float32x2_t)
+
+/*
+** test_vfmsq_lane_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f64, float64x2_t, float64x2_t, float64x2_t, float64x1_t)
+
+/*
+** test_vfmsq_laneq_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f16, float16x8_t, float16x8_t, float16x8_t, float16x8_t)
+
+/*
+** test_vfmsq_laneq_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f32, float32x4_t, float32x4_t, float32x4_t, float32x4_t)
+
+/*
+** test_vfmsq_laneq_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f64, float64x2_t, float64x2_t, float64x2_t, float64x2_t)
+
+/*
+** test_vfmsh_lane_f16:
+** fmls h0, h1, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmsh_lane_f16, float16_t, float16_t, float16_t, float16x4_t)
+
+/*
+** test_vfmsh_laneq_f16:
+** fmls h0, h1, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmsh_laneq_f16, float16_t, float16_t, float16_t, float16x8_t)
+
+/*
+** test_vfmss_lane_f32:
+** fmls s0, s1, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmss_lane_f32, float32_t, float32_t, float32_t, float32x2_t)
+
+/*
+** test_vfmss_laneq_f32:
+** fmls s0, s1, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmss_laneq_f32, float32_t, float32_t, float32_t, float32x4_t)
+
+/*
+** test_vfmsd_lane_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfmsd_lane_f64, float64_t, float64_t, float64_t, float64x1_t)
+
+/*
+** test_vfmsd_laneq_f64:
+** fmls d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmsd_laneq_f64, float64_t, float64_t, float64_t, float64x2_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
b/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
index c751924e8d1..cc2b3887947 100644
--- a/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/simd/pr99195_3.c
@@ -3,7 +3,7 @@
the top half of the vector register and no explicit zeroing instructions
are emitted. */
/* { dg-do compile } */
-/* { dg-options "-O" } */
+/* { dg-options "-O -fno-trapping-math" } */
#include <arm_neon.h>
@@ -65,4 +65,3 @@ OPTHREE (float32, 2, 4, f32, mla_lane, fma_lane, fms_lane)
/* { dg-final { scan-assembler-not {\tfmov\t} } } */
/* { dg-final { scan-assembler-not {\tmov\t} } } */
-
--
2.56.0
