https://gcc.gnu.org/g:446234f72f2f1227af5983ff4bb62b3f27c6fbd7
commit r17-6076-g446234f72f2f1227af5983ff4bb62b3f27c6fbd7 Author: Ezra Sitorus <[email protected]> Date: Mon Nov 9 23:58:57 2026 +0000 aarch64: FEAT_F16F32MM, FEAT_F16MM, FEAT_SVE_B16MM support Add the AdvSIMD vmmlaq_f16 and vmmlaq_f32_f16 intrinsics, together with the SVE svmmla_f16 and svmmla_bf16 forms and their overloaded aliases. This patch also requires this fix in binutils: https://inbox.sourceware.org/binutils/[email protected]/ gcc/ChangeLog: * config/aarch64/aarch64-c.cc (aarch64_update_cpp_builtins): Add new __ARM_FEATURE__ macros. * config/aarch64/aarch64-neon-builtins-base.cc (vmmlaq_impl): New function_base class. (vmmlaq): Define new function base. * config/aarch64/aarch64-neon-builtins-base.def (vmmlaq): Define new NEON function. * config/aarch64/aarch64-simd.md (@aarch64_fmmlaq<mode>): New pattern. (@aarch64_<insn><mode>): Use VQ_HSF_F8_FMMLA. * config/aarch64/aarch64-sve-builtins-sve2.def (svmmla): Define new SVE functions for F16MM and SVE_B16MM. * config/aarch64/aarch64-sve2.md (@aarch64_sve_<sve_fp_op><SVE_FULL_HF_FMMLA:mode><SVE_FULL_HF_FMMLA:mode>): New pattern. * config/aarch64/aarch64.h (TARGET_F16F32MM, TARGET_F16MM, TARGET_SVE_B16MM): New macros. * config/aarch64/iterators.md (VQ_HSF_F16_FMMLA): New iterator. (VQ_HSF_F8_FMMLA): Rename from VDQ_HSF_FMMLA. (SVE_FULL_HF_FMMLA): New iterator. gcc/testsuite/ChangeLog: * gcc.target/aarch64/pragma_cpp_predefs_4.c: Test new feature macros. * gcc.target/aarch64/sve/acle/general-c/mmla_1.c: Remove obsolete test * gcc.target/aarch64/sve/acle/general-c/mmla_8.c: New test. * gcc.target/aarch64/sve2/acle/asm/fmmla_bf16.c: New test. * gcc.target/aarch64/sve2/acle/asm/fmmla_f16.c: New test. * lib/target-supports.exp ("f16mm", "f16f32mm", "sve-b16mm"): Add assembly extension checks. Diff: --- gcc/config/aarch64/aarch64-c.cc | 3 +++ gcc/config/aarch64/aarch64-neon-builtins-base.cc | 13 +++++++++++ gcc/config/aarch64/aarch64-neon-builtins-base.def | 11 ++++++++- gcc/config/aarch64/aarch64-simd.md | 18 ++++++++++++--- gcc/config/aarch64/aarch64-sve-builtins-sve2.def | 10 ++++++++ gcc/config/aarch64/aarch64-sve2.md | 24 +++++++++++++++++++ gcc/config/aarch64/aarch64.h | 6 +++++ gcc/config/aarch64/iterators.md | 14 ++++++++--- .../gcc.target/aarch64/pragma_cpp_predefs_4.c | 15 ++++++++++++ .../gcc.target/aarch64/sve/acle/general-c/mmla_1.c | 1 - .../gcc.target/aarch64/sve/acle/general-c/mmla_8.c | 10 ++++++++ .../gcc.target/aarch64/sve2/acle/asm/fmmla_bf16.c | 27 ++++++++++++++++++++++ .../gcc.target/aarch64/sve2/acle/asm/fmmla_f16.c | 27 ++++++++++++++++++++++ gcc/testsuite/lib/target-supports.exp | 6 ++--- 14 files changed, 174 insertions(+), 11 deletions(-) diff --git a/gcc/config/aarch64/aarch64-c.cc b/gcc/config/aarch64/aarch64-c.cc index 09b3b53e1fb1..6ed87c8dbb70 100644 --- a/gcc/config/aarch64/aarch64-c.cc +++ b/gcc/config/aarch64/aarch64-c.cc @@ -347,6 +347,9 @@ aarch64_update_cpp_builtins (cpp_reader *pfile) aarch64_def_or_undef (TARGET_F8F16MM, "__ARM_FEATURE_F8F16MM", pfile); aarch64_def_or_undef (TARGET_F8F32MM, "__ARM_FEATURE_F8F32MM", pfile); + aarch64_def_or_undef (TARGET_F16MM, "__ARM_FEATURE_F16MM", pfile); + aarch64_def_or_undef (TARGET_F16F32MM, "__ARM_FEATURE_F16F32MM", pfile); + aarch64_def_or_undef (TARGET_SVE_B16MM, "__ARM_FEATURE_SVE_B16MM", pfile); aarch64_def_or_undef (TARGET_SVE_F16F32MM, "__ARM_FEATURE_SVE_F16F32MM", pfile); } diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.cc b/gcc/config/aarch64/aarch64-neon-builtins-base.cc index a20490775a99..4dd26fd102da 100644 --- a/gcc/config/aarch64/aarch64-neon-builtins-base.cc +++ b/gcc/config/aarch64/aarch64-neon-builtins-base.cc @@ -946,6 +946,16 @@ struct gimple_abs_diff : public gimple_function_base } }; +/* FP16 matrix-multiply accumulate (widening and non-widening). */ +class vmmlaq_impl : public function_base +{ +public: + rtx expand (function_expander &e) const override + { + return e.use_exact_insn (code_for_aarch64_fmmlaq (e.result_mode ())); + } +}; + // Reinterpret NEON_FUNCTION (vreinterpret, vreinterpretq, gimple_reinterpret,) @@ -1092,4 +1102,7 @@ NEON_FUNCTION (vsm3partw2q, gimple_exact_insn<CODE_FOR_aarch64_sm3partw2qv4si>,) // SM4 NEON_FUNCTION (vsm4eq, gimple_exact_insn<CODE_FOR_aarch64_sm4eqv4si>,) NEON_FUNCTION (vsm4ekeyq, gimple_exact_insn<CODE_FOR_aarch64_sm4ekeyqv4si>,) + +// FP16 matrix-multiply accumulate +NEON_FUNCTION (vmmlaq, vmmlaq_impl,) } diff --git a/gcc/config/aarch64/aarch64-neon-builtins-base.def b/gcc/config/aarch64/aarch64-neon-builtins-base.def index 353744c352ed..a42a25135a0c 100644 --- a/gcc/config/aarch64/aarch64-neon-builtins-base.def +++ b/gcc/config/aarch64/aarch64-neon-builtins-base.def @@ -395,4 +395,13 @@ DEF_NEON_FUNCTION (vsm3partw2q, s_unsigned, ("Q0,Q0,Q0,Q0")) #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_SIMD | AARCH64_FL_SM4) DEF_NEON_FUNCTION (vsm4eq, s_unsigned, ("Q0,Q0,Q0")) DEF_NEON_FUNCTION (vsm4ekeyq, s_unsigned, ("Q0,Q0,Q0")) -#undef REQUIRED_EXTENSIONS \ No newline at end of file +#undef REQUIRED_EXTENSIONS +// FP16 matrix-multiply accumulate (widening) +#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_F16F32MM) +DEF_NEON_FUNCTION (vmmlaq, cvt_f32_f16, ("Q0,Q0,Q1,Q1")) +#undef REQUIRED_EXTENSIONS + +// FP16 matrix-multiply accumulate (non-widening) +#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_F16MM) +DEF_NEON_FUNCTION (vmmlaq, h_float, ("Q0,Q0,Q0,Q0")) +#undef REQUIRED_EXTENSIONS diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index 1599f1fa349e..7ea023633139 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -10925,6 +10925,18 @@ [(set_attr "type" "neon_fp_mla_s_q")] ) +;; fmmla +(define_insn "@aarch64_fmmlaq<mode>" + [(set (match_operand:VQ_HSF_F16_FMMLA 0 "register_operand" "=w") + (plus:VQ_HSF_F16_FMMLA (match_operand:VQ_HSF_F16_FMMLA 1 "register_operand" "0") + (unspec:VQ_HSF_F16_FMMLA [(match_operand:V8HF 2 "register_operand" "w") + (match_operand:V8HF 3 "register_operand" "w")] + UNSPEC_FMMLA)))] + "" + "fmmla\t%0.<Vtype>, %2.8h, %3.8h" + [(set_attr "type" "neon_fp_mla_s_q")] +) + ;; bfmlal<bt> (define_insn "aarch64_bfmlal<bt>v4sf" [(set (match_operand:V4SF 0 "register_operand" "=w") @@ -11229,11 +11241,11 @@ ) (define_insn "@aarch64_<insn><mode>" - [(set (match_operand:VDQ_HSF_FMMLA 0 "register_operand") - (unspec:VDQ_HSF_FMMLA + [(set (match_operand:VQ_HSF_F8_FMMLA 0 "register_operand") + (unspec:VQ_HSF_F8_FMMLA [(match_operand:V16QI 2 "register_operand") (match_operand:V16QI 3 "register_operand") - (match_operand:VDQ_HSF_FMMLA 1 "register_operand") + (match_operand:VQ_HSF_F8_FMMLA 1 "register_operand") (reg:DI FPM_REGNUM)] FMMLA))] "" diff --git a/gcc/config/aarch64/aarch64-sve-builtins-sve2.def b/gcc/config/aarch64/aarch64-sve-builtins-sve2.def index 3e9d709f2b85..6ed9243c8dc0 100644 --- a/gcc/config/aarch64/aarch64-sve-builtins-sve2.def +++ b/gcc/config/aarch64/aarch64-sve-builtins-sve2.def @@ -496,6 +496,16 @@ DEF_SVE_FUNCTION_GS_FPM (svmmla, mmla, s_float_mf8, none, none, set) DEF_SVE_FUNCTION (svmmla, mmla, cvt_f32_f16, none) #undef REQUIRED_EXTENSIONS +#define REQUIRED_EXTENSIONS \ + nonstreaming_sve (AARCH64_FL_SVE_B16MM) +DEF_SVE_FUNCTION_GS (svmmla, mmla, h_bfloat, none, none) +#undef REQUIRED_EXTENSIONS + +#define REQUIRED_EXTENSIONS \ + nonstreaming_sve (AARCH64_FL_SVE2p2 | AARCH64_FL_F16MM) +DEF_SVE_FUNCTION_GS (svmmla, mmla, h_float, none, none) +#undef REQUIRED_EXTENSIONS + /* - BFSCALE (predicated) // Only if __ARM_FEATURE_SVE_BFSCALE != 0 && __ARM_FEATURE_SVE2 != 0 diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md index ab09875e329e..1e575fa50fd9 100644 --- a/gcc/config/aarch64/aarch64-sve2.md +++ b/gcc/config/aarch64/aarch64-sve2.md @@ -145,6 +145,7 @@ ;; ;; == FMMLA extensions ;; ---- [FP] Matrix multiply-accumulate widening +;; ---- [FP] Matrix multiply-accumulate non-widening ;; ========================================================================= ;; == Moves @@ -5092,3 +5093,26 @@ } [(set_attr "sve_type" "sve_fp_mul")] ) + +;; ------------------------------------------------------------------------- +;; ---- [FP] Matrix multiply-accumulate non-widening +;; ------------------------------------------------------------------------- +;; Includes: +;; - FMMLA (F16MM) +;; - BFMMLA (SVE_B16MM) +;; ------------------------------------------------------------------------- + +(define_insn "@aarch64_sve_<sve_fp_op><SVE_FULL_HF_FMMLA:mode><SVE_FULL_HF_FMMLA:mode>" + [(set (match_operand:SVE_FULL_HF_FMMLA 0 "register_operand") + (unspec:SVE_FULL_HF_FMMLA + [(match_operand:SVE_FULL_HF_FMMLA 2 "register_operand") + (match_operand:SVE_FULL_HF_FMMLA 3 "register_operand") + (match_operand:SVE_FULL_HF_FMMLA 1 "register_operand")] + FMMLA))] + "TARGET_NON_STREAMING" + {@ [ cons: =0 , 1 , 2 , 3 ; attrs: movprfx ] + [ w , 0 , w , w ; * ] <b><sve_fp_op>\t%0.h, %2.h, %3.h + [ ?&w , w , w , w ; yes ] movprfx\t%0, %1\;<b><sve_fp_op>\t%0.h, %2.h, %3.h + } + [(set_attr "sve_type" "sve_fp_mul")] +) diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h index 39f51d4e92d7..0174944d362b 100644 --- a/gcc/config/aarch64/aarch64.h +++ b/gcc/config/aarch64/aarch64.h @@ -438,10 +438,16 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED #define TARGET_F8F32MM (AARCH64_HAVE_ISA (F8F32MM)) /* F8F16MM instructions, enabled through +f8f16mm. */ #define TARGET_F8F16MM (AARCH64_HAVE_ISA (F8F16MM)) +/* F16F32MM instructions, enabled through +f16f32mm. */ +#define TARGET_F16F32MM (AARCH64_HAVE_ISA (F16F32MM)) +/* F16MM instructions, enabled through +f16mm. */ +#define TARGET_F16MM (AARCH64_HAVE_ISA (F16MM)) /* SVE_F16F32MM instructions, enabled through +sve-f16f32mm. */ #define TARGET_SVE_F16F32MM (AARCH64_HAVE_ISA (SVE_F16F32MM)) /* F16F32DOT instructions enabled through +f16f32dot. */ #define TARGET_F16F32DOT (AARCH64_HAVE_ISA (F16F32DOT)) +/* SVE_B16MM instructions, enabled through +sve-b16mm. */ +#define TARGET_SVE_B16MM (AARCH64_HAVE_ISA (SVE_B16MM)) /* Make sure this is always defined so we don't have to check for ifdefs but rather use normal ifs. */ diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index dfee954a5d54..b18094d9088d 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -226,8 +226,12 @@ (V2SF "TARGET_FP8DOT4") (V4SF "TARGET_FP8DOT4")]) -(define_mode_iterator VDQ_HSF_FMMLA [(V8HF "TARGET_F8F16MM") - (V4SF "TARGET_F8F32MM")]) +;; Same, but with the appropriate conditions for FMMLA support with FP16 source. +(define_mode_iterator VQ_HSF_F16_FMMLA [(V8HF "TARGET_F16MM") + (V4SF "TARGET_F16F32MM")]) + +(define_mode_iterator VQ_HSF_F8_FMMLA [(V8HF "TARGET_F8F16MM") + (V4SF "TARGET_F8F32MM")]) ;; Modes suitable to use as the return type of a vcond expression. (define_mode_iterator VDQF_COND [V2SF V2SI V4SF V4SI V2DF V2DI]) @@ -596,7 +600,11 @@ ;; Fully-packed SVE vector modes that have 16-bit float elements. (define_mode_iterator SVE_FULL_HF [VNx8BF VNx8HF]) -;; Pairs of the above. +;; Same, but with the appropriate conditions for FMMLA support. +(define_mode_iterator SVE_FULL_HF_FMMLA [(VNx8BF "TARGET_SVE_B16MM") + (VNx8HF "TARGET_F16MM && TARGET_SVE2p2")]) + +;; Pairs of SVE_FULL_HF. (define_mode_iterator SVE_FULL_HFx2 [VNx16BF VNx16HF]) ;; {f16}{x1,x2} diff --git a/gcc/testsuite/gcc.target/aarch64/pragma_cpp_predefs_4.c b/gcc/testsuite/gcc.target/aarch64/pragma_cpp_predefs_4.c index 4caa01bba0f8..9ba60c2be1ec 100644 --- a/gcc/testsuite/gcc.target/aarch64/pragma_cpp_predefs_4.c +++ b/gcc/testsuite/gcc.target/aarch64/pragma_cpp_predefs_4.c @@ -430,3 +430,18 @@ #ifdef __ARM_FEATURE_CSSC #error Foo #endif + +#pragma GCC target "+nothing+f16mm" +#ifndef __ARM_FEATURE_F16MM +#error Foo +#endif + +#pragma GCC target "+nothing+f16f32mm" +#ifndef __ARM_FEATURE_F16F32MM +#error Foo +#endif + +#pragma GCC target "+nothing+sve-b16mm" +#ifndef __ARM_FEATURE_SVE_B16MM +#error Foo +#endif diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general-c/mmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general-c/mmla_1.c index a23c45c86b14..c616ad425c40 100644 --- a/gcc/testsuite/gcc.target/aarch64/sve/acle/general-c/mmla_1.c +++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general-c/mmla_1.c @@ -52,7 +52,6 @@ f2 (svbool_t pg, svint8_t s8, svuint8_t u8, svuint32_t u32, svint32_t s32, svmmla (f64, f32, f16); /* { dg-error {'svmmla' has no form that takes 'svfloat64_t' and 'svfloat32_t' arguments} } */ svmmla (f64, f64, f16); /* { dg-error {passing 'svfloat16_t' to argument 3 of 'svmmla', but argument 2 had type 'svfloat64_t'} } */ - svmmla (f16, f16, f16); /* { dg-error {'svmmla' has no form that takes 'svfloat16_t' arguments} } */ svmmla (f32, f32, f32); svmmla (f64, f64, f64); } diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general-c/mmla_8.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general-c/mmla_8.c new file mode 100644 index 000000000000..430e4378bf0e --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general-c/mmla_8.c @@ -0,0 +1,10 @@ +/* { dg-do compile } */ +/* { dg-additional-options "-march=armv8.2-a+sve" } */ + +#include <arm_sve.h> + +void +f1 (svfloat16_t f16) +{ + svmmla (f16, f16, f16); /* { dg-error {ACLE function 'svmmla_f16' requires ISA extension 'sve2p2'} } */ +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/asm/fmmla_bf16.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/asm/fmmla_bf16.c new file mode 100644 index 000000000000..2e8a62218fad --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/asm/fmmla_bf16.c @@ -0,0 +1,27 @@ +/* { dg-do assemble { target aarch64_asm_sve-b16mm_ok } } */ +/* { dg-do compile { target { ! aarch64_asm_sve-b16mm_ok } } } */ +/* { dg-skip-if "" { *-*-* } { "-DSTREAMING_COMPATIBLE" } { "" } } */ +/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */ + +#include "test_sve_acle.h" + +#pragma GCC target "+sve-b16mm" + +/* +** svmmla_bf16mm_tied: +** bfmmla z0\.h, z4\.h, z5\.h +** ret +*/ +TEST_DUAL_Z (svmmla_bf16mm_tied, svbfloat16_t, svbfloat16_t, + z0 = svmmla_bf16 (z0, z4, z5), + z0 = svmmla (z0, z4, z5)) + +/* +** svmmla_bf16mm: +** movprfx z0, z1 +** bfmmla z0\.h, z4\.h, z5\.h +** ret +*/ +TEST_DUAL_Z (svmmla_bf16mm, svbfloat16_t, svbfloat16_t, + z0 = svmmla_bf16 (z1, z4, z5), + z0 = svmmla (z1, z4, z5)) diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/acle/asm/fmmla_f16.c b/gcc/testsuite/gcc.target/aarch64/sve2/acle/asm/fmmla_f16.c new file mode 100644 index 000000000000..b5adf846d227 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve2/acle/asm/fmmla_f16.c @@ -0,0 +1,27 @@ +/* { dg-do assemble { target { aarch64_asm_sve2p2_ok && aarch64_asm_f16mm_ok } } } */ +/* { dg-do compile { target { ! { aarch64_asm_sve2p2_ok && aarch64_asm_f16mm_ok } } } } */ +/* { dg-skip-if "" { *-*-* } { "-DSTREAMING_COMPATIBLE" } { "" } } */ +/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */ + +#include "test_sve_acle.h" + +#pragma GCC target "+sve2p2+f16mm" + +/* +** svmmla_f16mm_tied: +** fmmla z0\.h, z4\.h, z5\.h +** ret +*/ +TEST_DUAL_Z (svmmla_f16mm_tied, svfloat16_t, svfloat16_t, + z0 = svmmla_f16 (z0, z4, z5), + z0 = svmmla (z0, z4, z5)) + +/* +** svmmla_f16mm: +** movprfx z0, z1 +** fmmla z0\.h, z4\.h, z5\.h +** ret +*/ +TEST_DUAL_Z (svmmla_f16mm, svfloat16_t, svfloat16_t, + z0 = svmmla_f16 (z1, z4, z5), + z0 = svmmla (z1, z4, z5)) diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp index 9e69020e78aa..5a102323c01d 100644 --- a/gcc/testsuite/lib/target-supports.exp +++ b/gcc/testsuite/lib/target-supports.exp @@ -13390,9 +13390,9 @@ proc check_effective_target_aarch64_gas_has_dtprel_reloc { } { set exts { "bf16" "cmpbr" "crc" "crypto" "dotprod" "f32mm" "f64mm" "faminmax" "fp" "fp8" "fp8dot2" "fp8dot4" "fp8fma" "i8mm" "ls64" "lse" "lut" - "sb" "simd" "sve-b16b16" "sve" "sve2" "sve-sm4" "sve-aes" "sve-aes2" - "sve-bitperm" "sve-sha3" "f8f16mm" "f8f32mm" "sve-f16f32mm" - "sme-f8f16" "sme-f8f32" + "sb" "simd" "f16mm" "f16f32mm" "sve-b16b16" "sve" "sve2" "sve-sm4" + "sve-aes" "sve-aes2" "sve-bitperm" "sve-sha3" "f8f16mm" "f8f32mm" + "sve-f16f32mm" "sve-b16mm" "sme-f8f16" "sme-f8f32" "sme-b16b16" "sme-f16f16" "sme-i16i64" "sme" "sme2" "sme2p1" "sme2p2" "sme2p3" "sme-tmop"
