gcc/ChangeLog:
* config/aarch64/aarch64-neon-builtins-base.cc
(singleton_vector_p): New function.
(class gimple_fused_mla): New class.
* config/aarch64/aarch64-neon-builtins-base.def
(vfma, vfmaq, vfma_n, vfmaq_n, vfma_lane, vfma_laneq,
vfmaq_lane, vfmaq_laneq, vfms, vfmsq, vfms_n, vfmsq_n,
vfms_lane, vfms_laneq, vfmsq_lane, vfmsq_laneq): New function
declarations.
* config/aarch64/aarch64-simd.md (fmav1df4<vczle><vczbe>,
fnmav1df4<vczle><vczbe>): New insns.
(*aarch64_fma4_elt<mode><vczle><vczbe>): Match against HF
modes as well.
(*aarch64_fma4_elt_<vswap_width_name><mode><vczle><vczbe>):
Likewise.
(*aarch64_fnma4_elt<mode><vczle><vczbe>): Match against HF
modes, and permute first two argument operands to match the RTL
generated by `.FNMA`.
(*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>):
Likewise.
(*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>): Likewise.
(*aarch64_fnma4_elt_to_64v2df): Likewise.
* config/aarch64/arm_neon.h (vfma_f64, vfma_f32, vfmaq_f32,
vfmaq_f64, vfma_n_f32, vfma_n_f64, vfmaq_n_f32, vfmaq_n_f64,
vfma_lane_f32, vfma_lane_f64, vfma_laneq_f32, vfma_laneq_f64,
vfmaq_lane_f32, vfmaq_lane_f64, vfmaq_laneq_f32,
vfmaq_laneq_f64, vfms_f64, vfms_f32, vfmsq_f32, vfmsq_f64,
vfms_n_f32, vfms_n_f64, vfmsq_n_f32, vfmsq_n_f64,
vfms_lane_f32, vfms_lane_f64, vfms_laneq_f32, vfms_laneq_f64,
vfmsq_lane_f32, vfmsq_lane_f64, vfmsq_laneq_f32,
vfmsq_laneq_f64, vfma_f16, vfmaq_f16, vfms_f16, vfmsq_f16,
vfma_lane_f16, vfmaq_lane_f16, vfma_laneq_f16,
vfmaq_laneq_f16, vfma_n_f16, vfmaq_n_f16, vfms_lane_f16,
vfmsq_lane_f16, vfms_laneq_f16, vfmsq_laneq_f16, vfms_n_f16,
vfmsq_n_f16): Delete function definitions.
* config/aarch64/iterators.md (VDQHSF): New mode iterator.
* config/arm/types.md (neon_fp_mla_h, neon_fp_mla_h_q,
neon_fp_mla_h_scalar, neon_fp_mla_h_scalar_q): New attrs.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/fmla_intrinsic_1.c: Rewrite `TEST_VMLA`
macro so that the choice of lane is always valid and doesn't
cause a compiler error when trying to call `vfma`. Also remove
excessive backslashes from dejagnu directives.
* gcc.target/aarch64/fmls_intrinsic_1.c: Likewise.
* gcc.target/aarch64/neon/vfma.c: New test.
* gcc.target/aarch64/neon/vfms.c: New test.
---
.../aarch64/aarch64-neon-builtins-base.cc | 98 +++-
.../aarch64/aarch64-neon-builtins-base.def | 54 ++
gcc/config/aarch64/aarch64-simd.md | 122 +++--
gcc/config/aarch64/arm_neon.h | 500 ------------------
gcc/config/aarch64/iterators.md | 7 +
gcc/config/arm/types.md | 20 +-
.../gcc.target/aarch64/fmla_intrinsic_1.c | 25 +-
.../gcc.target/aarch64/fmls_intrinsic_1.c | 25 +-
gcc/testsuite/gcc.target/aarch64/neon/vfma.c | 220 ++++++++
gcc/testsuite/gcc.target/aarch64/neon/vfms.c | 220 ++++++++
10 files changed, 717 insertions(+), 574 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vfma.c
create mode 100644 gcc/testsuite/gcc.target/aarch64/neon/vfms.c
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
index 37071c4aab8..ee29905e63b 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc
@@ -45,6 +45,15 @@
#include "gimple-fold.h"
namespace aarch64_acle {
+/* Return true if VEC (a vector expression or vector type) is known to have a
+ single element. */
+bool
+singleton_vector_p (tree vec)
+{
+ tree type = TYPE_P (vec) ? vec : TREE_TYPE (vec);
+ return VECTOR_TYPE_P (type) && known_eq (TYPE_VECTOR_SUBPARTS (type), 1U);
+}
+
/* Build a cast expression, `(TYPE)EXPR`, if necessary to make an expression
with type TYPE. */
tree
@@ -101,7 +110,7 @@ build_lane_set (tree vec, tree index, tree elem)
tree
build_vec_dup (tree type, tree elem)
{
- return known_eq (TYPE_VECTOR_SUBPARTS (type), 1U)
+ return singleton_vector_p (type)
? fold_build1 (VIEW_CONVERT_EXPR, type, elem)
: fold_build1 (VEC_DUPLICATE_EXPR, type, elem);
}
@@ -224,7 +233,7 @@ struct gimple_combine : public gimple_function_base
auto elem_type = TREE_TYPE (arg_type);
auto ret_type = TREE_TYPE (f.lhs);
- if (known_eq (TYPE_VECTOR_SUBPARTS (arg_type), 1U))
+ if (singleton_vector_p (arg_type))
{
arg1 = f.force_val (fold_build1 (VIEW_CONVERT_EXPR, elem_type, arg1));
arg2 = f.force_val (fold_build1 (VIEW_CONVERT_EXPR, elem_type, arg2));
@@ -759,6 +768,72 @@ public:
}
};
+enum class fma_kind
+{
+ normal,
+ dup,
+ lane,
+};
+
+template <fma_kind kind, bool negate>
+class gimple_fused_mla : public gimple_function_base
+{
+public:
+ gimple *fold (gimple_folder &f) const override
+ {
+ tree add = gimple_call_arg (f.call, 0);
+ tree mul1 = gimple_call_arg (f.call, 1);
+ tree mul2 = gimple_call_arg (f.call, 2);
+ tree lane_index;
+ tree ret_type = TREE_TYPE (f.lhs);
+ tree ret = f.lhs;
+
+ auto mk_call = [] (auto mul1, auto mul2, auto add) {
+ return gimple_build_call_internal (negate ? IFN_FNMA : IFN_FMA, 3, mul1,
+ mul2, add);
+ };
+
+ switch (kind)
+ {
+ case fma_kind::normal:
+ break;
+
+ case fma_kind::dup:
+ mul2 = f.force_val (build_vec_dup (TREE_TYPE (mul1), mul2));
+ break;
+
+ case fma_kind::lane:
+ lane_index = gimple_call_arg (f.call, 3);
+ mul2 = f.force_val (build_lane_get (mul2, lane_index));
+
+ /* FIXME (Karl Meakin): Hack to get better codegen for `vmla_laneq_f64`.
+ Instead of building an unecessary VECTOR_DUP, call FMA with the
+ element type and then cast the result to a singleton vector. */
+ if (singleton_vector_p (ret_type) || SCALAR_FLOAT_TYPE_P (ret_type))
+ {
+ tree elem_type = TREE_TYPE (mul2);
+ add = f.force_val (build_cast (elem_type, add));
+ mul1 = f.force_val (build_cast (elem_type, mul1));
+ ret = create_tmp_var (elem_type);
+ auto call = mk_call (mul1, mul2, add);
+ gsi_insert_before (f.gsi, call, GSI_SAME_STMT);
+ gimple_call_set_lhs (call, ret);
+ return gimple_build_assign (f.lhs, f.force_val (
+ build_cast (ret_type, ret)));
+ }
+ else
+ mul2 = f.force_val (build_vec_dup (TREE_TYPE (mul1), mul2));
+ break;
+ }
+
+ auto call = mk_call (mul1, mul2, add);
+ gimple_call_set_lhs (call, ret);
+ return call;
+ }
+};
+template <fma_kind kind> using gimple_fma = gimple_fused_mla<kind, false>;
+template <fma_kind kind> using gimple_fms = gimple_fused_mla<kind, true>;
+
// Reinterpret
NEON_FUNCTION (vreinterpret, vreinterpretq, gimple_reinterpret,)
@@ -783,6 +858,25 @@ NEON_FUNCTION (vmla, vmlaq, vmla_n, vmlaq_n, vmla_lane,
vmla_laneq, vmlaq_lane,
NEON_FUNCTION (vmls, vmlsq, vmls_n, vmlsq_n, vmls_lane, vmls_laneq, vmlsq_lane,
vmlsq_laneq, gimple_mla, (MINUS_EXPR))
+// Multiply-accumulate (fused)
+NEON_FUNCTION (vfma, vfmaq, gimple_fma<fma_kind::normal>,);
+NEON_FUNCTION (vfma_n, vfmaq_n, gimple_fma<fma_kind::dup>,);
+NEON_FUNCTION (vfmah_lane, vfmah_laneq,
+ vfmas_lane, vfmas_laneq,
+ vfmad_lane, vfmad_laneq,
+ vfma_lane, vfma_laneq,
+ vfmaq_lane, vfmaq_laneq,
+ gimple_fma<fma_kind::lane>,);
+
+NEON_FUNCTION (vfms, vfmsq, gimple_fms<fma_kind::normal>,);
+NEON_FUNCTION (vfms_n, vfmsq_n, gimple_fms<fma_kind::dup>,);
+NEON_FUNCTION (vfmsh_lane, vfmsh_laneq,
+ vfmss_lane, vfmss_laneq,
+ vfmsd_lane, vfmsd_laneq,
+ vfms_lane, vfms_laneq,
+ vfmsq_lane, vfmsq_laneq,
+ gimple_fms<fma_kind::lane>,);
+
// Bitwise operations
NEON_FUNCTION (vand, vandq, gimple_expr, (BIT_AND_EXPR))
NEON_FUNCTION (vbic, vbicq, gimple_not_rhs, (BIT_AND_EXPR))
diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.def
b/gcc/config/aarch64/aarch64-neon-builtins-base.def
index d2ab46a3d95..49380ca24ec 100644
--- a/gcc/config/aarch64/aarch64-neon-builtins-base.def
+++ b/gcc/config/aarch64/aarch64-neon-builtins-base.def
@@ -96,6 +96,60 @@ DEF_NEON_FUNCTION (vmlsq_lane, neon_mla_lane,
("Q0,Q0,Q0,D0,ss32", lane<3>))
DEF_NEON_FUNCTION (vmlsq_laneq, neon_mla_lane, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
#undef REQUIRED_EXTENSIONS
+// Fused multiply-accumulate
+#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
+DEF_NEON_FUNCTION (vfma, sd_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmaq, sd_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfma_n, sd_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmaq_n, sd_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfma_lane, sd_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfma_laneq, sd_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_lane, sd_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_laneq, sd_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmas_lane, s_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmas_laneq, s_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmad_lane, d_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmad_laneq, d_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+
+DEF_NEON_FUNCTION (vfms, sd_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmsq, sd_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfms_n, sd_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmsq_n, sd_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfms_lane, sd_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfms_laneq, sd_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_lane, sd_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_laneq, sd_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmss_lane, s_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmss_laneq, s_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsd_lane, d_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsd_laneq, d_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+#undef REQUIRED_EXTENSIONS
+
+// Fused multiply-accumulate (FP16)
+#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD |
AARCH64_FL_F16)
+DEF_NEON_FUNCTION (vfma, h_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmaq, h_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfma_n, h_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmaq_n, h_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfma_lane, h_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfma_laneq, h_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_lane, h_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmaq_laneq, h_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmah_lane, h_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmah_laneq, h_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+
+DEF_NEON_FUNCTION (vfms, h_float, ("D0,D0,D0,D0"))
+DEF_NEON_FUNCTION (vfmsq, h_float, ("Q0,Q0,Q0,Q0"))
+DEF_NEON_FUNCTION (vfms_n, h_float, ("D0,D0,D0,s0"))
+DEF_NEON_FUNCTION (vfmsq_n, h_float, ("Q0,Q0,Q0,s0"))
+DEF_NEON_FUNCTION (vfms_lane, h_float, ("D0,D0,D0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfms_laneq, h_float, ("D0,D0,D0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_lane, h_float, ("Q0,Q0,Q0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsq_laneq, h_float, ("Q0,Q0,Q0,Q0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsh_lane, h_float, ("s0,s0,s0,D0,ss32", lane<3>))
+DEF_NEON_FUNCTION (vfmsh_laneq, h_float, ("s0,s0,s0,Q0,ss32", lane<3>))
+#undef REQUIRED_EXTENSIONS
+
// Bitwise operations
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD)
DEF_NEON_FUNCTION (vand, all_integer, ("D0,D0,D0"))
diff --git a/gcc/config/aarch64/aarch64-simd.md
b/gcc/config/aarch64/aarch64-simd.md
index 0b824e73dbf..ba1b87e0a9a 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -3050,15 +3050,48 @@ (define_insn "fma<mode>4<vczle><vczbe>"
[(set_attr "type" "neon_fp_mla_<stype><q>")]
)
+(define_insn "fmav1df4<vczle><vczbe>"
+ [(set (match_operand:V1DF 0 "register_operand" "=w")
+ (fma:V1DF (match_operand:V1DF 3 "register_operand" "w")
+ (match_operand:V1DF 1 "register_operand" "%w")
+ (match_operand:V1DF 2 "register_operand" "w")))]
+ "TARGET_SIMD"
+ "fmadd\\t%d0, %d1, %d2, %d3"
+ [(set_attr "type" "neon_fp_mla_d_q")]
+)
+
+(define_insn "fnmav1df4<vczle><vczbe>"
+ [(set (match_operand:V1DF 0 "register_operand" "=w")
+ (fma:V1DF (neg:V1DF (match_operand:V1DF 3 "register_operand" "w"))
+ (match_operand:V1DF 1 "register_operand" "%w")
+ (match_operand:V1DF 2 "register_operand" "w")))]
+ "TARGET_SIMD"
+ "fmsub\\t%d0, %d1, %d2, %d3"
+ [(set_attr "type" "neon_fp_mla_d_q")]
+)
+
+(define_insn "*aarch64_fma<mode>4_elt_to_scalar"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))
+ (match_operand:<VEL> 3 "register_operand" "w")
+ (match_operand:<VEL> 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ "fmla\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]"
+ [(set_attr "type" "neon_fp_mla_d_q")]
+)
+
(define_insn "*aarch64_fma4_elt<mode><vczle><vczbe>"
- [(set (match_operand:VDQF 0 "register_operand" "=w")
- (fma:VDQF
- (vec_duplicate:VDQF
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (vec_duplicate:VHSDF
(vec_select:<VEL>
- (match_operand:VDQF 1 "register_operand" "<h_con>")
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
(parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQF 3 "register_operand" "w")
- (match_operand:VDQF 4 "register_operand" "0")))]
+ (match_operand:VHSDF 3 "register_operand" "w")
+ (match_operand:VHSDF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
@@ -3068,14 +3101,14 @@ (define_insn "*aarch64_fma4_elt<mode><vczle><vczbe>"
)
(define_insn "*aarch64_fma4_elt_<vswap_width_name><mode><vczle><vczbe>"
- [(set (match_operand:VDQSF 0 "register_operand" "=w")
- (fma:VDQSF
- (vec_duplicate:VDQSF
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (vec_duplicate:VDQHSF
(vec_select:<VEL>
(match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
(parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQSF 3 "register_operand" "w")
- (match_operand:VDQSF 4 "register_operand" "0")))]
+ (match_operand:VDQHSF 3 "register_operand" "w")
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
@@ -3102,8 +3135,8 @@ (define_insn "*aarch64_fma4_elt_to_64v2df"
(vec_select:DF
(match_operand:V2DF 1 "register_operand" "w")
(parallel [(match_operand:SI 2 "immediate_operand")]))
- (match_operand:DF 3 "register_operand" "w")
- (match_operand:DF 4 "register_operand" "0")))]
+ (match_operand:DF 3 "register_operand" "w")
+ (match_operand:DF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
@@ -3124,15 +3157,14 @@ (define_insn "fnma<mode>4<vczle><vczbe>"
)
(define_insn "*aarch64_fnma4_elt<mode><vczle><vczbe>"
- [(set (match_operand:VDQF 0 "register_operand" "=w")
- (fma:VDQF
- (neg:VDQF
- (match_operand:VDQF 3 "register_operand" "w"))
- (vec_duplicate:VDQF
+ [(set (match_operand:VHSDF 0 "register_operand" "=w")
+ (fma:VHSDF
+ (neg:VHSDF (vec_duplicate:VHSDF
(vec_select:<VEL>
- (match_operand:VDQF 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQF 4 "register_operand" "0")))]
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")]))))
+ (match_operand:VHSDF 3 "register_operand" "w")
+ (match_operand:VHSDF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
@@ -3141,16 +3173,28 @@ (define_insn "*aarch64_fnma4_elt<mode><vczle><vczbe>"
[(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
)
+(define_insn "*aarch64_fnma<mode>4_elt_to_scalar"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+ (fma:<VEL>
+ (neg:<VEL> (vec_select:<VEL>
+ (match_operand:VHSDF 1 "register_operand" "<h_con>")
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:<VEL> 3 "register_operand" "w")
+ (match_operand:<VEL> 4 "register_operand" "0")))]
+ "TARGET_SIMD"
+ "fmls\\t%<Vetype>0, %<Vetype>3, %1.<Vetype>[%2]"
+ [(set_attr "type" "neon_fp_mla_d_q")]
+)
+
(define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>"
- [(set (match_operand:VDQSF 0 "register_operand" "=w")
- (fma:VDQSF
- (neg:VDQSF
- (match_operand:VDQSF 3 "register_operand" "w"))
- (vec_duplicate:VDQSF
+ [(set (match_operand:VDQHSF 0 "register_operand" "=w")
+ (fma:VDQHSF
+ (neg:VDQHSF (vec_duplicate:VDQHSF
(vec_select:<VEL>
(match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
- (parallel [(match_operand:SI 2 "immediate_operand")])))
- (match_operand:VDQSF 4 "register_operand" "0")))]
+ (parallel [(match_operand:SI 2 "immediate_operand")]))))
+ (match_operand:VDQHSF 3 "register_operand" "w")
+ (match_operand:VDQHSF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL
(operands[2]));
@@ -3162,8 +3206,19 @@ (define_insn
"*aarch64_fnma4_elt_<vswap_width_name><mode><vczle><vczbe>"
(define_insn "*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>"
[(set (match_operand:VMUL 0 "register_operand" "=w")
(fma:VMUL
- (neg:VMUL
- (match_operand:VMUL 2 "register_operand" "w"))
+ (neg:VMUL (vec_duplicate:VMUL
+ (match_operand:<VEL> 1 "register_operand" "<h_con>")))
+ (match_operand:VMUL 2 "register_operand" "w")
+ (match_operand:VMUL 3 "register_operand" "0")))]
+ "TARGET_SIMD"
+ "fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
+ [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
+)
+
+(define_insn "*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>_permuted"
+ [(set (match_operand:VMUL 0 "register_operand" "=w")
+ (fma:VMUL
+ (neg:VMUL (match_operand:VMUL 2 "register_operand" "w"))
(vec_duplicate:VMUL
(match_operand:<VEL> 1 "register_operand" "<h_con>"))
(match_operand:VMUL 3 "register_operand" "0")))]
@@ -3175,11 +3230,10 @@ (define_insn
"*aarch64_fnma4_elt_from_dup<mode><vczle><vczbe>"
(define_insn "*aarch64_fnma4_elt_to_64v2df"
[(set (match_operand:DF 0 "register_operand" "=w")
(fma:DF
- (vec_select:DF
+ (neg:DF (vec_select:DF
(match_operand:V2DF 1 "register_operand" "w")
- (parallel [(match_operand:SI 2 "immediate_operand")]))
- (neg:DF
- (match_operand:DF 3 "register_operand" "w"))
+ (parallel [(match_operand:SI 2 "immediate_operand")])))
+ (match_operand:DF 3 "register_operand" "w")
(match_operand:DF 4 "register_operand" "0")))]
"TARGET_SIMD"
{
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index 968609fcd4b..b499ed9361d 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -7990,352 +7990,6 @@ vcvtpq_u64_f64 (float64x2_t __a)
return __builtin_aarch64_lceiluv2dfv2di_us (__a);
}
-/* vfma */
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f64 (float64x1_t __a, float64x1_t __b, float64x1_t __c)
-{
- return (float64x1_t) {__builtin_fma (__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f32 (float32x2_t __a, float32x2_t __b, float32x2_t __c)
-{
- return __builtin_aarch64_fmav2sf (__b, __c, __a);
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f32 (float32x4_t __a, float32x4_t __b, float32x4_t __c)
-{
- return __builtin_aarch64_fmav4sf (__b, __c, __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f64 (float64x2_t __a, float64x2_t __b, float64x2_t __c)
-{
- return __builtin_aarch64_fmav2df (__b, __c, __a);
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f32 (float32x2_t __a, float32x2_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav2sf (__b, vdup_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f64 (float64x1_t __a, float64x1_t __b, float64_t __c)
-{
- return (float64x1_t) {__b[0] * __c + __a[0]};
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f32 (float32x4_t __a, float32x4_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav4sf (__b, vdupq_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f64 (float64x2_t __a, float64x2_t __b, float64_t __c)
-{
- return __builtin_aarch64_fmav2df (__b, vdupq_n_f64 (__c), __a);
-}
-
-/* vfma_lane */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f32 (float32x2_t __a, float32x2_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (__b,
- __aarch64_vdup_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f64 (float64x1_t __a, float64x1_t __b,
- float64x1_t __c, const int __lane)
-{
- return (float64x1_t) {__builtin_fma (__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmad_lane_f64 (float64_t __a, float64_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_fma (__b, __c[0], __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmas_lane_f32 (float32_t __a, float32_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_fmaf (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfma_laneq */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f32 (float32x2_t __a, float32x2_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (__b,
- __aarch64_vdup_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f64 (float64x1_t __a, float64x1_t __b,
- float64x2_t __c, const int __lane)
-{
- float64_t __c0 = __aarch64_vget_lane_any (__c, __lane);
- return (float64x1_t) {__builtin_fma (__b[0], __c0, __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmad_laneq_f64 (float64_t __a, float64_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_fma (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmas_laneq_f32 (float32_t __a, float32_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_fmaf (__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfmaq_lane */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f32 (float32x4_t __a, float32x4_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (__b,
- __aarch64_vdupq_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f64 (float64x2_t __a, float64x2_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (__b, vdupq_n_f64 (__c[0]), __a);
-}
-
-/* vfmaq_laneq */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f32 (float32x4_t __a, float32x4_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (__b,
- __aarch64_vdupq_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f64 (float64x2_t __a, float64x2_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (__b,
- __aarch64_vdupq_laneq_f64 (__c, __lane),
- __a);
-}
-
-/* vfms */
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f64 (float64x1_t __a, float64x1_t __b, float64x1_t __c)
-{
- return (float64x1_t) {__builtin_fma (-__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f32 (float32x2_t __a, float32x2_t __b, float32x2_t __c)
-{
- return __builtin_aarch64_fmav2sf (-__b, __c, __a);
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f32 (float32x4_t __a, float32x4_t __b, float32x4_t __c)
-{
- return __builtin_aarch64_fmav4sf (-__b, __c, __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f64 (float64x2_t __a, float64x2_t __b, float64x2_t __c)
-{
- return __builtin_aarch64_fmav2df (-__b, __c, __a);
-}
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f32 (float32x2_t __a, float32x2_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav2sf (-__b, vdup_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f64 (float64x1_t __a, float64x1_t __b, float64_t __c)
-{
- return (float64x1_t) {-__b[0] * __c + __a[0]};
-}
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f32 (float32x4_t __a, float32x4_t __b, float32_t __c)
-{
- return __builtin_aarch64_fmav4sf (-__b, vdupq_n_f32 (__c), __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f64 (float64x2_t __a, float64x2_t __b, float64_t __c)
-{
- return __builtin_aarch64_fmav2df (-__b, vdupq_n_f64 (__c), __a);
-}
-
-/* vfms_lane */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f32 (float32x2_t __a, float32x2_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (-__b,
- __aarch64_vdup_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f64 (float64x1_t __a, float64x1_t __b,
- float64x1_t __c, const int __lane)
-{
- return (float64x1_t) {__builtin_fma (-__b[0], __c[0], __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsd_lane_f64 (float64_t __a, float64_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_fma (-__b, __c[0], __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmss_lane_f32 (float32_t __a, float32_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_fmaf (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfms_laneq */
-
-__extension__ extern __inline float32x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f32 (float32x2_t __a, float32x2_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2sf (-__b,
- __aarch64_vdup_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x1_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f64 (float64x1_t __a, float64x1_t __b,
- float64x2_t __c, const int __lane)
-{
- float64_t __c0 = __aarch64_vget_lane_any (__c, __lane);
- return (float64x1_t) {__builtin_fma (-__b[0], __c0, __a[0])};
-}
-
-__extension__ extern __inline float64_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsd_laneq_f64 (float64_t __a, float64_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_fma (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-__extension__ extern __inline float32_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmss_laneq_f32 (float32_t __a, float32_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_fmaf (-__b, __aarch64_vget_lane_any (__c, __lane), __a);
-}
-
-/* vfmsq_lane */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f32 (float32x4_t __a, float32x4_t __b,
- float32x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (-__b,
- __aarch64_vdupq_lane_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f64 (float64x2_t __a, float64x2_t __b,
- float64x1_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (-__b, vdupq_n_f64 (__c[0]), __a);
-}
-
-/* vfmsq_laneq */
-
-__extension__ extern __inline float32x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f32 (float32x4_t __a, float32x4_t __b,
- float32x4_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav4sf (-__b,
- __aarch64_vdupq_laneq_f32 (__c, __lane),
- __a);
-}
-
-__extension__ extern __inline float64x2_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f64 (float64x2_t __a, float64x2_t __b,
- float64x2_t __c, const int __lane)
-{
- return __builtin_aarch64_fmav2df (-__b,
- __aarch64_vdupq_laneq_f64 (__c, __lane),
- __a);
-}
-
/* vld1 */
__extension__ extern __inline float16x4_t
@@ -19852,162 +19506,8 @@ vsubq_f16 (float16x8_t __a, float16x8_t __b)
return __a - __b;
}
-/* ARMv8.2-A FP16 three operands vector intrinsics. */
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_f16 (float16x4_t __a, float16x4_t __b, float16x4_t __c)
-{
- return __builtin_aarch64_fmav4hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_f16 (float16x8_t __a, float16x8_t __b, float16x8_t __c)
-{
- return __builtin_aarch64_fmav8hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_f16 (float16x4_t __a, float16x4_t __b, float16x4_t __c)
-{
- return __builtin_aarch64_fnmav4hf (__b, __c, __a);
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_f16 (float16x8_t __a, float16x8_t __b, float16x8_t __c)
-{
- return __builtin_aarch64_fnmav8hf (__b, __c, __a);
-}
-
/* ARMv8.2-A FP16 lane vector intrinsics. */
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmah_lane_f16 (float16_t __a, float16_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmah_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmah_laneq_f16 (float16_t __a, float16_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmah_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_lane_f16 (float16x4_t __a, float16x4_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfma_f16 (__a, __b, __aarch64_vdup_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_lane_f16 (float16x8_t __a, float16x8_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmaq_f16 (__a, __b, __aarch64_vdupq_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_laneq_f16 (float16x4_t __a, float16x4_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfma_f16 (__a, __b, __aarch64_vdup_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_laneq_f16 (float16x8_t __a, float16x8_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmaq_f16 (__a, __b, __aarch64_vdupq_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfma_n_f16 (float16x4_t __a, float16x4_t __b, float16_t __c)
-{
- return vfma_f16 (__a, __b, vdup_n_f16 (__c));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmaq_n_f16 (float16x8_t __a, float16x8_t __b, float16_t __c)
-{
- return vfmaq_f16 (__a, __b, vdupq_n_f16 (__c));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsh_lane_f16 (float16_t __a, float16_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmsh_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsh_laneq_f16 (float16_t __a, float16_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmsh_f16 (__a, __b, __aarch64_vget_lane_any (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_lane_f16 (float16x4_t __a, float16x4_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfms_f16 (__a, __b, __aarch64_vdup_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_lane_f16 (float16x8_t __a, float16x8_t __b,
- float16x4_t __c, const int __lane)
-{
- return vfmsq_f16 (__a, __b, __aarch64_vdupq_lane_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_laneq_f16 (float16x4_t __a, float16x4_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfms_f16 (__a, __b, __aarch64_vdup_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_laneq_f16 (float16x8_t __a, float16x8_t __b,
- float16x8_t __c, const int __lane)
-{
- return vfmsq_f16 (__a, __b, __aarch64_vdupq_laneq_f16 (__c, __lane));
-}
-
-__extension__ extern __inline float16x4_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfms_n_f16 (float16x4_t __a, float16x4_t __b, float16_t __c)
-{
- return vfms_f16 (__a, __b, vdup_n_f16 (__c));
-}
-
-__extension__ extern __inline float16x8_t
-__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
-vfmsq_n_f16 (float16x8_t __a, float16x8_t __b, float16_t __c)
-{
- return vfmsq_f16 (__a, __b, vdupq_n_f16 (__c));
-}
-
__extension__ extern __inline float16_t
__attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
vmulh_lane_f16 (float16_t __a, float16x4_t __b, const int __lane)
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 568786e3714..001a4734394 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -215,6 +215,13 @@ (define_mode_iterator VSFDF [V2SF V4SF V2DF DF SF])
;; Advanced SIMD single Float modes.
(define_mode_iterator VDQSF [V2SF V4SF])
+;; Advanced SIMD half and single Float modes.
+(define_mode_iterator VDQHSF [
+ (V4HF "TARGET_SIMD_F16INST")
+ (V8HF "TARGET_SIMD_F16INST")
+ V2SF V4SF
+])
+
;; Quad vector float modes with half/bfloat elements.
(define_mode_iterator VQ_BHF [V8HF V8BF])
diff --git a/gcc/config/arm/types.md b/gcc/config/arm/types.md
index 7f8d83be410..45ec40f81c0 100644
--- a/gcc/config/arm/types.md
+++ b/gcc/config/arm/types.md
@@ -471,6 +471,10 @@ (define_attr "autodetect_type"
; neon_fp_mul_d
; neon_fp_mul_d_q
; neon_fp_mul_d_scalar_q
+; neon_fp_mla_h
+; neon_fp_mla_h_q
+; neon_fp_mla_h_scalar
+; neon_fp_mla_h_scalar_q
; neon_fp_mla_s
; neon_fp_mla_s_q
; neon_fp_mla_s_scalar
@@ -969,6 +973,10 @@ (define_attr "type"
neon_fp_mul_d_q,\
neon_fp_mul_d_scalar_q,\
\
+ neon_fp_mla_h,\
+ neon_fp_mla_h_q,\
+ neon_fp_mla_h_scalar,\
+ neon_fp_mla_h_scalar_q,\
neon_fp_mla_s,\
neon_fp_mla_s_q,\
neon_fp_mla_s_scalar,\
@@ -1154,12 +1162,12 @@ (define_attr "is_neon_type" "yes,no"
neon_fp_recpx_d_q, neon_fp_rsqrte_s, neon_fp_rsqrte_s_q,\
neon_fp_rsqrte_d, neon_fp_rsqrte_d_q, neon_fp_rsqrts_s,\
neon_fp_rsqrts_s_q, neon_fp_rsqrts_d, neon_fp_rsqrts_d_q,\
- neon_fp_mul_s, neon_fp_mul_s_q, neon_fp_mul_s_scalar,\
- neon_fp_mul_s_scalar_q, neon_fp_mul_d, neon_fp_mul_d_q,\
- neon_fp_mul_d_scalar_q, neon_fp_mla_s, neon_fp_mla_s_q,\
- neon_fp_mla_s_scalar, neon_fp_mla_s_scalar_q, neon_fp_mla_d,\
- neon_fp_mla_d_q, neon_fp_mla_d_scalar_q, neon_fp_sqrt_s,\
- neon_fp_sqrt_s_q, neon_fp_sqrt_d, neon_fp_sqrt_d_q,\
+ neon_fp_mul_s, neon_fp_mul_s_q, neon_fp_mul_s_scalar,
neon_fp_mul_s_scalar_q,\
+ neon_fp_mul_d, neon_fp_mul_d_q,
neon_fp_mul_d_scalar_q,\
+ neon_fp_mla_h, neon_fp_mla_h_q, neon_fp_mla_h_scalar,
neon_fp_mla_h_scalar_q,\
+ neon_fp_mla_s, neon_fp_mla_s_q, neon_fp_mla_s_scalar,
neon_fp_mla_s_scalar_q,\
+ neon_fp_mla_d, neon_fp_mla_d_q,
neon_fp_mla_d_scalar_q,\
+ neon_fp_sqrt_s, neon_fp_sqrt_s_q, neon_fp_sqrt_d, neon_fp_sqrt_d_q,\
neon_fp_div_s, neon_fp_div_s_q, neon_fp_div_d, neon_fp_div_d_q,
crypto_aese,\
neon_fcadd, neon_fcmla, \
crypto_aesmc, crypto_sha1_xor, crypto_sha1_fast, crypto_sha1_slow,\
diff --git a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
index c1aeb06e747..9571f6df727 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
@@ -19,16 +19,9 @@ test_vfma##q1##_lane##q2##_f##size (float##size##_t * res,
\
float##size##x##in1_lanes##_t a = vld1##q1##_f##size (res); \
float##size##x##in1_lanes##_t b = vld1##q1##_f##size (in1); \
float##size##x##in2_lanes##_t c; \
- if (in2_lanes > 1) \
- { \
- c = vld1##q2##_f##size (in2); \
- a = vfma##q1##_lane##q2##_f##size (a, b, c, 1); \
- } \
- else \
- { \
- c = vld1##q2##_f##size (in2 + 1);
\
- a = vfma##q1##_lane##q2##_f##size (a, b, c, 0); \
- } \
+ const int lane_no = in2_lanes > 1 ? 1 : 0; \
+ c = vld1##q2##_f##size (in2 + (in2_lanes == 1)); \
+ a = vfma##q1##_lane##q2##_f##size (a, b, c, lane_no);
\
vst1##q1##_f##size (res, a); \
}
@@ -99,20 +92,20 @@ main (int argc, char **argv)
/* vfma_laneq_f32.
vfma_lane_f32. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2s, v[0-9]+.2s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfmaq_lane_f32.
vfmaq_laneq_f32. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.4s, v[0-9]+.4s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfma_lane_f64.
vfma_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 1 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmadd d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmadd d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 2 { target aarch64_little_endian } } } */
/* vfmaq_lane_f64.
vfmaq_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmla v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 2 { target aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
index 3137ea91e80..0f7e1a76487 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
@@ -19,16 +19,9 @@ test_vfms##q1##_lane##q2##_f##size (float##size##_t * res,
\
float##size##x##in1_lanes##_t a = vld1##q1##_f##size (res); \
float##size##x##in1_lanes##_t b = vld1##q1##_f##size (in1); \
float##size##x##in2_lanes##_t c; \
- if (in2_lanes > 1) \
- { \
- c = vld1##q2##_f##size (in2); \
- a = vfms##q1##_lane##q2##_f##size (a, b, c, 1); \
- } \
- else \
- { \
- c = vld1##q2##_f##size (in2 + 1);
\
- a = vfms##q1##_lane##q2##_f##size (a, b, c, 0); \
- } \
+ const int lane_no = in2_lanes > 1 ? 1 : 0; \
+ c = vld1##q2##_f##size (in2 + (in2_lanes == 1)); \
+ a = vfms##q1##_lane##q2##_f##size (a, b, c, lane_no);
\
vst1##q1##_f##size (res, a); \
}
@@ -100,20 +93,20 @@ main (int argc, char **argv)
/* vfms_laneq_f32.
vfms_lane_f32. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2s, v[0-9]+.2s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfmsq_lane_f32.
vfmsq_laneq_f32. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s,
v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.4s, v[0-9]+.4s,
v[0-9]+.s\[[0-9]+\]} 2 } } */
/* vfms_lane_f64.
vfms_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 1 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\,
d\[0-9\]+\, d\[0-9\]+" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmsub d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmsub d[0-9]+, d[0-9]+, d[0-9]+,
d[0-9]+} 2 { target aarch64_little_endian } } } */
/* vfmsq_lane_f64.
vfmsq_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d,
v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times {fmls v[0-9]+.2d, v[0-9]+.2d,
v[0-9]+.d\[[0-9]+\]} 2 { target aarch64_little_endian } } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
b/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
new file mode 100644
index 00000000000..7de3e90b867
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vfma.c
@@ -0,0 +1,220 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+#define TEST_LANE(NAME, RET_TYPE, TYPE1, TYPE2, TYPE3)
\
+ RET_TYPE test_##NAME (TYPE1 a, TYPE2 b, TYPE3 c)
\
+ {
\
+ return NAME (a, b, c, VEC_LEN (c) - 1);
\
+ }
+
+/*
+** test_vfma_f16:
+** fmla v0\.4h, (v1\.4h, v2\.4h|v2\.4h, v1\.4h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f16, float16x4_t)
+
+/*
+** test_vfmaq_f16:
+** fmla v0\.8h, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f16, float16x8_t)
+
+/*
+** test_vfma_f32:
+** fmla v0\.2s, (v1\.2s, v2\.2s|v2\.2s, v1\.2s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f32, float32x2_t)
+
+/*
+** test_vfmaq_f32:
+** fmla v0\.4s, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f32, float32x4_t)
+
+/*
+** test_vfma_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfma_f64, float64x1_t)
+
+/*
+** test_vfmaq_f64:
+** fmla v0\.2d, (v1\.2d, v2\.2d|v2\.2d, v1\.2d)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmaq_f64, float64x2_t)
+
+/*
+** test_vfma_n_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfma_n_f16, float16x4_t, float16x4_t, float16x4_t, float16_t)
+
+/*
+** test_vfmaq_n_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f16, float16x8_t, float16x8_t, float16x8_t, float16_t)
+
+/*
+** test_vfma_n_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfma_n_f32, float32x2_t, float32x2_t, float32x2_t, float32_t)
+
+/*
+** test_vfmaq_n_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f32, float32x4_t, float32x4_t, float32x4_t, float32_t)
+
+/*
+** test_vfma_n_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_TERNARY (vfma_n_f64, float64x1_t, float64x1_t, float64x1_t, float64_t)
+
+/*
+** test_vfmaq_n_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_TERNARY (vfmaq_n_f64, float64x2_t, float64x2_t, float64x2_t, float64_t)
+
+/*
+** test_vfma_lane_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfma_lane_f16, float16x4_t, float16x4_t, float16x4_t, float16x4_t)
+
+/*
+** test_vfma_lane_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfma_lane_f32, float32x2_t, float32x2_t, float32x2_t, float32x2_t)
+
+/*
+** test_vfma_lane_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfma_lane_f64, float64x1_t, float64x1_t, float64x1_t, float64x1_t)
+
+/*
+** test_vfma_laneq_f16:
+** fmla v0\.4h, v1\.4h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f16, float16x4_t, float16x4_t, float16x4_t, float16x8_t)
+
+/*
+** test_vfma_laneq_f32:
+** fmla v0\.2s, v1\.2s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f32, float32x2_t, float32x2_t, float32x2_t, float32x4_t)
+
+/*
+** test_vfma_laneq_f64:
+** fmla d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfma_laneq_f64, float64x1_t, float64x1_t, float64x1_t, float64x2_t)
+
+/*
+** test_vfmaq_lane_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f16, float16x8_t, float16x8_t, float16x8_t, float16x4_t)
+
+/*
+** test_vfmaq_lane_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f32, float32x4_t, float32x4_t, float32x4_t, float32x2_t)
+
+/*
+** test_vfmaq_lane_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_LANE (vfmaq_lane_f64, float64x2_t, float64x2_t, float64x2_t, float64x1_t)
+
+/*
+** test_vfmaq_laneq_f16:
+** fmla v0\.8h, v1\.8h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f16, float16x8_t, float16x8_t, float16x8_t, float16x8_t)
+
+/*
+** test_vfmaq_laneq_f32:
+** fmla v0\.4s, v1\.4s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f32, float32x4_t, float32x4_t, float32x4_t, float32x4_t)
+
+/*
+** test_vfmaq_laneq_f64:
+** fmla v0\.2d, v1\.2d, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmaq_laneq_f64, float64x2_t, float64x2_t, float64x2_t, float64x2_t)
+
+/*
+** test_vfmah_lane_f16:
+** fmla h0, h1, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmah_lane_f16, float16_t, float16_t, float16_t, float16x4_t)
+
+/*
+** test_vfmah_laneq_f16:
+** fmla h0, h1, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmah_laneq_f16, float16_t, float16_t, float16_t, float16x8_t)
+
+/*
+** test_vfmas_lane_f32:
+** fmla s0, s1, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmas_lane_f32, float32_t, float32_t, float32_t, float32x2_t)
+
+/*
+** test_vfmas_laneq_f32:
+** fmla s0, s1, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmas_laneq_f32, float32_t, float32_t, float32_t, float32x4_t)
+
+/*
+** test_vfmad_lane_f64:
+** fmadd d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfmad_lane_f64, float64_t, float64_t, float64_t, float64x1_t)
+
+/*
+** test_vfmad_laneq_f64:
+** fmla d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmad_laneq_f64, float64_t, float64_t, float64_t, float64x2_t)
diff --git a/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
b/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
new file mode 100644
index 00000000000..c5bd09d3a61
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neon/vfms.c
@@ -0,0 +1,220 @@
+/* { dg-do compile } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon_test.h"
+
+#define TEST_LANE(NAME, RET_TYPE, TYPE1, TYPE2, TYPE3)
\
+ RET_TYPE test_##NAME (TYPE1 a, TYPE2 b, TYPE3 c)
\
+ {
\
+ return NAME (a, b, c, VEC_LEN (c) - 1);
\
+ }
+
+/*
+** test_vfms_f16:
+** fmls v0\.4h, (v1\.4h, v2\.4h|v2\.4h, v1\.4h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f16, float16x4_t)
+
+/*
+** test_vfmsq_f16:
+** fmls v0\.8h, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f16, float16x8_t)
+
+/*
+** test_vfms_f32:
+** fmls v0\.2s, (v1\.2s, v2\.2s|v2\.2s, v1\.2s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f32, float32x2_t)
+
+/*
+** test_vfmsq_f32:
+** fmls v0\.4s, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f32, float32x4_t)
+
+/*
+** test_vfms_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfms_f64, float64x1_t)
+
+/*
+** test_vfmsq_f64:
+** fmls v0\.2d, (v1\.2d, v2\.2d|v2\.2d, v1\.2d)
+** ret
+*/
+TEST_UNIFORM_TERNARY (vfmsq_f64, float64x2_t)
+
+/*
+** test_vfms_n_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfms_n_f16, float16x4_t, float16x4_t, float16x4_t, float16_t)
+
+/*
+** test_vfmsq_n_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f16, float16x8_t, float16x8_t, float16x8_t, float16_t)
+
+/*
+** test_vfms_n_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfms_n_f32, float32x2_t, float32x2_t, float32x2_t, float32_t)
+
+/*
+** test_vfmsq_n_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f32, float32x4_t, float32x4_t, float32x4_t, float32_t)
+
+/*
+** test_vfms_n_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_TERNARY (vfms_n_f64, float64x1_t, float64x1_t, float64x1_t, float64_t)
+
+/*
+** test_vfmsq_n_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_TERNARY (vfmsq_n_f64, float64x2_t, float64x2_t, float64x2_t, float64_t)
+
+/*
+** test_vfms_lane_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfms_lane_f16, float16x4_t, float16x4_t, float16x4_t, float16x4_t)
+
+/*
+** test_vfms_lane_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfms_lane_f32, float32x2_t, float32x2_t, float32x2_t, float32x2_t)
+
+/*
+** test_vfms_lane_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfms_lane_f64, float64x1_t, float64x1_t, float64x1_t, float64x1_t)
+
+/*
+** test_vfms_laneq_f16:
+** fmls v0\.4h, v1\.4h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f16, float16x4_t, float16x4_t, float16x4_t, float16x8_t)
+
+/*
+** test_vfms_laneq_f32:
+** fmls v0\.2s, v1\.2s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f32, float32x2_t, float32x2_t, float32x2_t, float32x4_t)
+
+/*
+** test_vfms_laneq_f64:
+** fmls d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfms_laneq_f64, float64x1_t, float64x1_t, float64x1_t, float64x2_t)
+
+/*
+** test_vfmsq_lane_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f16, float16x8_t, float16x8_t, float16x8_t, float16x4_t)
+
+/*
+** test_vfmsq_lane_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f32, float32x4_t, float32x4_t, float32x4_t, float32x2_t)
+
+/*
+** test_vfmsq_lane_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[0\]
+** ret
+*/
+TEST_LANE (vfmsq_lane_f64, float64x2_t, float64x2_t, float64x2_t, float64x1_t)
+
+/*
+** test_vfmsq_laneq_f16:
+** fmls v0\.8h, v1\.8h, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f16, float16x8_t, float16x8_t, float16x8_t, float16x8_t)
+
+/*
+** test_vfmsq_laneq_f32:
+** fmls v0\.4s, v1\.4s, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f32, float32x4_t, float32x4_t, float32x4_t, float32x4_t)
+
+/*
+** test_vfmsq_laneq_f64:
+** fmls v0\.2d, v1\.2d, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmsq_laneq_f64, float64x2_t, float64x2_t, float64x2_t, float64x2_t)
+
+/*
+** test_vfmsh_lane_f16:
+** fmls h0, h1, v2\.h\[3\]
+** ret
+*/
+TEST_LANE (vfmsh_lane_f16, float16_t, float16_t, float16_t, float16x4_t)
+
+/*
+** test_vfmsh_laneq_f16:
+** fmls h0, h1, v2\.h\[7\]
+** ret
+*/
+TEST_LANE (vfmsh_laneq_f16, float16_t, float16_t, float16_t, float16x8_t)
+
+/*
+** test_vfmss_lane_f32:
+** fmls s0, s1, v2\.s\[1\]
+** ret
+*/
+TEST_LANE (vfmss_lane_f32, float32_t, float32_t, float32_t, float32x2_t)
+
+/*
+** test_vfmss_laneq_f32:
+** fmls s0, s1, v2\.s\[3\]
+** ret
+*/
+TEST_LANE (vfmss_laneq_f32, float32_t, float32_t, float32_t, float32x4_t)
+
+/*
+** test_vfmsd_lane_f64:
+** fmsub d0, (d1, d2|d2, d1), d0
+** ret
+*/
+TEST_LANE (vfmsd_lane_f64, float64_t, float64_t, float64_t, float64x1_t)
+
+/*
+** test_vfmsd_laneq_f64:
+** fmls d0, d1, v2\.d\[1\]
+** ret
+*/
+TEST_LANE (vfmsd_laneq_f64, float64_t, float64_t, float64_t, float64x2_t)
--
2.51.0
