Changes from v1: Fixed formatting issues in changelog. This patch depends on: 1. Support for Dense Math Registers: https://gcc.gnu.org/pipermail/gcc-patches/2026-April/713352.html 2. Support for new DMF/MMA+ instructions: https://gcc.gnu.org/pipermail/gcc-patches/2026-June/721366.html
This patch adds builtins for the Matrix Multiply Assist Plus (MMA+) instructions defined in RFC02658. These instructions may or may not be supported in a future Power processor. Specifically, builtins have been added for the following instructions: 1. Bfloat16 GER operations: - dmxvbf16gerx2 and its variants (pp, pn, np, nn) - pmdmxvbf16gerx2 and its variants (pp, pn, np, nn) 2. Signed accumulate variant for 8-bit integer GER operations: - dmxvi8gerx4spp - pmdmxvi8gerx4spp Note, the names of the builtins may change in the future. 2026-06-24 Surya Kumari Jangala <[email protected]> gcc: * config/rs6000/mma.md (UNSPEC_DMF_DMXVI8GERX4SPP): New UNSPEC entry. (UNSPEC_DMF_PMDMXVI8GERX4SPP): Likewise. (UNSPEC_DMF_DMXVBF16GERX2): Likewise. (UNSPEC_DMF_DMXVBF16GERX2PP): Likewise. (UNSPEC_DMF_DMXVBF16GERX2PN): Likewise. (UNSPEC_DMF_DMXVBF16GERX2NP): Likewise. (UNSPEC_DMF_DMXVBF16GERX2NN): Likewise. (UNSPEC_DMF_PMDMXVBF16GERX2): Likewise. (UNSPEC_DMF_PMDMXVBF16GERX2PP): Likewise. (UNSPEC_DMF_PMDMXVBF16GERX2PN): Likewise. (UNSPEC_DMF_PMDMXVBF16GERX2NP): Likewise. (UNSPEC_DMF_PMDMXVBF16GERX2NN): Likewise. (DMF_PV): Add UNSPEC_DMF_DMXVBF16GERX2. (DMF_DPV): Add bf16 and int8 signed accumulate variants. (DMF_DPVI8I4I4): Add UNSPEC_DMF_PMDMXVI8GERX4SPP. (DMF_PVI8I4I2): New int iterator. (DMF_DPVI8I4I2): Likewise. (pv): Add dmxvbf16gerx2 attribute. (apv): Removed dmxvi8gerx4pp attribute. (dpv): New attribute. (dpvi8i4i4): Add pmdmxvi8gerx4spp attribute. (pvi8i4i2): New attribute. (dpvi8i4i2): Likewise. (dmf_<dpv>): Rename from dmf_<apv> and update to use dpv attribute. (dmf_<pvi8i4i2>): New insn pattern. (dmf_<dpvi8i4i2>): Likewise. * config/rs6000/rs6000-builtins.def (__builtin_mma_dmxvi8gerx4spp): New builtin. (__builtin_mma_pmdmxvi8gerx4spp): Likewise. (__builtin_mma_dmxvbf16gerx2): Likewise. (__builtin_mma_dmxvbf16gerx2pp): Likewise. (__builtin_mma_dmxvbf16gerx2pn): Likewise. (__builtin_mma_dmxvbf16gerx2np): Likewise. (__builtin_mma_dmxvbf16gerx2nn): Likewise. (__builtin_mma_pmdmxvbf16gerx2): Likewise. (__builtin_mma_pmdmxvbf16gerx2pp): Likewise. (__builtin_mma_pmdmxvbf16gerx2pn): Likewise. (__builtin_mma_pmdmxvbf16gerx2np): Likewise. (__builtin_mma_pmdmxvbf16gerx2nn): Likewise. (__builtin_mma_dmxvi8gerx4spp_internal): New internal builtin. (__builtin_mma_pmdmxvi8gerx4spp_internal): Likewise. (__builtin_mma_dmxvbf16gerx2_internal): Likewise. (__builtin_mma_dmxvbf16gerx2pp_internal): Likewise. (__builtin_mma_dmxvbf16gerx2pn_internal): Likewise. (__builtin_mma_dmxvbf16gerx2np_internal): Likewise. (__builtin_mma_dmxvbf16gerx2nn_internal): Likewise. (__builtin_mma_pmdmxvbf16gerx2_internal): Likewise. (__builtin_mma_pmdmxvbf16gerx2pp_internal): Likewise. (__builtin_mma_pmdmxvbf16gerx2pn_internal): Likewise. (__builtin_mma_pmdmxvbf16gerx2np_internal): Likewise. (__builtin_mma_pmdmxvbf16gerx2nn_internal): Likewise. * doc/extend.texi (PowerPC Matrix-Multiply Assist Built-in Functions): Document new MMA+ builtins for int8 and bf16 operations. gcc/testsuite: * gcc.target/powerpc/dmf-builtin-1.c: New test. --- gcc/config/rs6000/mma.md | 104 +++++++-- gcc/config/rs6000/rs6000-builtins.def | 85 ++++++++ gcc/doc/extend.texi | 14 ++ .../gcc.target/powerpc/dmf-builtin-1.c | 204 ++++++++++++++++++ 4 files changed, 393 insertions(+), 14 deletions(-) create mode 100644 gcc/testsuite/gcc.target/powerpc/dmf-builtin-1.c diff --git a/gcc/config/rs6000/mma.md b/gcc/config/rs6000/mma.md index 7ad6ad07647..7044c1e7056 100644 --- a/gcc/config/rs6000/mma.md +++ b/gcc/config/rs6000/mma.md @@ -100,8 +100,20 @@ (define_c_enum "unspec" UNSPEC_DMF_DMXOR UNSPEC_DMF_DMXVI8GERX4 UNSPEC_DMF_DMXVI8GERX4PP + UNSPEC_DMF_DMXVI8GERX4SPP UNSPEC_DMF_PMDMXVI8GERX4 UNSPEC_DMF_PMDMXVI8GERX4PP + UNSPEC_DMF_PMDMXVI8GERX4SPP + UNSPEC_DMF_DMXVBF16GERX2 + UNSPEC_DMF_DMXVBF16GERX2PP + UNSPEC_DMF_DMXVBF16GERX2PN + UNSPEC_DMF_DMXVBF16GERX2NP + UNSPEC_DMF_DMXVBF16GERX2NN + UNSPEC_DMF_PMDMXVBF16GERX2 + UNSPEC_DMF_PMDMXVBF16GERX2PP + UNSPEC_DMF_PMDMXVBF16GERX2PN + UNSPEC_DMF_PMDMXVBF16GERX2NP + UNSPEC_DMF_PMDMXVBF16GERX2NN ]) (define_c_enum "unspecv" @@ -144,8 +156,9 @@ (define_int_iterator MMA_AVV [UNSPEC_MMA_XVI4GER8PP ;; MMA instructions with 1 vector pair and 1 vector arguments (define_int_iterator MMA_PV [UNSPEC_MMA_XVF64GER]) -; DMF instructions with 1 vector pair and 1 vector arguments -(define_int_iterator DMF_PV [UNSPEC_DMF_DMXVI8GERX4]) +;; DMF instructions with 1 vector pair and 1 vector arguments +(define_int_iterator DMF_PV [UNSPEC_DMF_DMXVI8GERX4 + UNSPEC_DMF_DMXVBF16GERX2]) ;; MMA instructions with 1 accumulator, 1 vector pair and 1 vector arguments (define_int_iterator MMA_APV [UNSPEC_MMA_XVF64GERPP @@ -154,7 +167,12 @@ (define_int_iterator MMA_APV [UNSPEC_MMA_XVF64GERPP UNSPEC_MMA_XVF64GERNN]) ;; DMF instructions with 1 dmr, 1 vector pair and 1 vector arguments -(define_int_iterator DMF_DPV [UNSPEC_DMF_DMXVI8GERX4PP]) +(define_int_iterator DMF_DPV [UNSPEC_DMF_DMXVI8GERX4PP + UNSPEC_DMF_DMXVI8GERX4SPP + UNSPEC_DMF_DMXVBF16GERX2PP + UNSPEC_DMF_DMXVBF16GERX2PN + UNSPEC_DMF_DMXVBF16GERX2NP + UNSPEC_DMF_DMXVBF16GERX2NN]) ;; MMA instructions with 2 vector, 2 4-bit and 1 8-bit arguments (define_int_iterator MMA_VVI4I4I8 [UNSPEC_MMA_PMXVI4GER8]) @@ -211,7 +229,19 @@ (define_int_iterator DMF_PVI8I4I4 [UNSPEC_DMF_PMDMXVI8GERX4]) ;; DMF instructions with 1dmr, 1 vector pair, 1 vector and 1 8-bit and ;; 2 4-bit arguments -(define_int_iterator DMF_DPVI8I4I4 [UNSPEC_DMF_PMDMXVI8GERX4PP]) +(define_int_iterator DMF_DPVI8I4I4 [UNSPEC_DMF_PMDMXVI8GERX4PP + UNSPEC_DMF_PMDMXVI8GERX4SPP]) + +;; DMF instructions with 1 vector pair, 1 vector, 1 8-bit, 1 4-bit +;; and 1 2-bit arguments +(define_int_iterator DMF_PVI8I4I2 [UNSPEC_DMF_PMDMXVBF16GERX2]) + +;; DMF instructions with 1dmr, 1 vector pair, 1 vector, 1 8-bit, +;; 1 4-bit and 1 2-bit arguments +(define_int_iterator DMF_DPVI8I4I2 [UNSPEC_DMF_PMDMXVBF16GERX2PP + UNSPEC_DMF_PMDMXVBF16GERX2PN + UNSPEC_DMF_PMDMXVBF16GERX2NP + UNSPEC_DMF_PMDMXVBF16GERX2NN]) (define_int_attr acc [(UNSPEC_MMA_XXMFACC "xxmfacc") (UNSPEC_MMA_XXMTACC "xxmtacc")]) @@ -243,13 +273,20 @@ (define_int_attr avv [(UNSPEC_MMA_XVI4GER8PP "xvi4ger8pp") (UNSPEC_MMA_XVF32GERNN "xvf32gernn")]) (define_int_attr pv [(UNSPEC_MMA_XVF64GER "xvf64ger") - (UNSPEC_DMF_DMXVI8GERX4 "dmxvi8gerx4")]) + (UNSPEC_DMF_DMXVI8GERX4 "dmxvi8gerx4") + (UNSPEC_DMF_DMXVBF16GERX2 "dmxvbf16gerx2")]) (define_int_attr apv [(UNSPEC_MMA_XVF64GERPP "xvf64gerpp") (UNSPEC_MMA_XVF64GERPN "xvf64gerpn") (UNSPEC_MMA_XVF64GERNP "xvf64gernp") - (UNSPEC_MMA_XVF64GERNN "xvf64gernn") - (UNSPEC_DMF_DMXVI8GERX4PP "dmxvi8gerx4pp")]) + (UNSPEC_MMA_XVF64GERNN "xvf64gernn")]) + +(define_int_attr dpv [(UNSPEC_DMF_DMXVI8GERX4PP "dmxvi8gerx4pp") + (UNSPEC_DMF_DMXVI8GERX4SPP "dmxvi8gerx4spp") + (UNSPEC_DMF_DMXVBF16GERX2PP "dmxvbf16gerx2pp") + (UNSPEC_DMF_DMXVBF16GERX2PN "dmxvbf16gerx2pn") + (UNSPEC_DMF_DMXVBF16GERX2NP "dmxvbf16gerx2np") + (UNSPEC_DMF_DMXVBF16GERX2NN "dmxvbf16gerx2nn")]) (define_int_attr vvi4i4i8 [(UNSPEC_MMA_PMXVI4GER8 "pmxvi4ger8")]) @@ -292,7 +329,15 @@ (define_int_attr avvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4PP "pmxvi8ger4pp") (define_int_attr pvi8i4i4 [(UNSPEC_DMF_PMDMXVI8GERX4 "pmdmxvi8gerx4")]) -(define_int_attr dpvi8i4i4 [(UNSPEC_DMF_PMDMXVI8GERX4PP "pmdmxvi8gerx4pp")]) +(define_int_attr dpvi8i4i4 [(UNSPEC_DMF_PMDMXVI8GERX4PP "pmdmxvi8gerx4pp") + (UNSPEC_DMF_PMDMXVI8GERX4SPP "pmdmxvi8gerx4spp")]) + +(define_int_attr pvi8i4i2 [(UNSPEC_DMF_PMDMXVBF16GERX2 "pmdmxvbf16gerx2")]) + +(define_int_attr dpvi8i4i2 [(UNSPEC_DMF_PMDMXVBF16GERX2PP "pmdmxvbf16gerx2pp") + (UNSPEC_DMF_PMDMXVBF16GERX2PN "pmdmxvbf16gerx2pn") + (UNSPEC_DMF_PMDMXVBF16GERX2NP "pmdmxvbf16gerx2np") + (UNSPEC_DMF_PMDMXVBF16GERX2NN "pmdmxvbf16gerx2nn")]) ;; Vector pair support. OOmode can only live in VSRs. @@ -1005,15 +1050,15 @@ (define_insn "dmf_<pv>" } [(set_attr "type" "dmf")]) -(define_insn "dmf_<apv>" +(define_insn "dmf_<dpv>" [(set (match_operand:TDO 0 "accumulator_operand" "=wD") - (unspec:TDO [(match_operand:TDO 1 "accumulator_operand" "0") - (match_operand:OO 2 "vsx_register_operand" "wa") - (match_operand:V16QI 3 "vsx_register_operand" "wa")] - DMF_DPV))] + (unspec:TDO [(match_operand:TDO 1 "accumulator_operand" "0") + (match_operand:OO 2 "vsx_register_operand" "wa") + (match_operand:V16QI 3 "vsx_register_operand" "wa")] + DMF_DPV))] "TARGET_DENSE_MATH" { - return "<apv> %0,%x2,%x3"; + return "<dpv> %0,%x2,%x3"; } [(set_attr "type" "dmf")]) @@ -1047,3 +1092,34 @@ (define_insn "dmf_<dpvi8i4i4>" } [(set_attr "type" "dmf") (set_attr "prefixed" "yes")]) + +(define_insn "dmf_<pvi8i4i2>" + [(set (match_operand:TDO 0 "accumulator_operand" "=wD") + (unspec:TDO [(match_operand:OO 1 "vsx_register_operand" "wa") + (match_operand:V16QI 2 "vsx_register_operand" "wa") + (match_operand:SI 3 "u8bit_cint_operand" "n") + (match_operand:SI 4 "const_0_to_15_operand" "n") + (match_operand:SI 5 "const_0_to_3_operand" "n")] + DMF_PVI8I4I2))] + "TARGET_DENSE_MATH" +{ + return "<pvi8i4i2> %0,%x1,%x2,%3,%4,%5"; +} + [(set_attr "type" "dmf") + (set_attr "prefixed" "yes")]) + +(define_insn "dmf_<dpvi8i4i2>" + [(set (match_operand:TDO 0 "accumulator_operand" "=wD") + (unspec:TDO [(match_operand:TDO 1 "accumulator_operand" "0") + (match_operand:OO 2 "vsx_register_operand" "wa") + (match_operand:V16QI 3 "vsx_register_operand" "wa") + (match_operand:SI 4 "u8bit_cint_operand" "n") + (match_operand:SI 5 "const_0_to_15_operand" "n") + (match_operand:SI 6 "const_0_to_3_operand" "n")] + DMF_DPVI8I4I2))] + "TARGET_DENSE_MATH" +{ + return "<dpvi8i4i2> %0,%x2,%x3,%4,%5,%6"; +} + [(set_attr "type" "dmf") + (set_attr "prefixed" "yes")]) diff --git a/gcc/config/rs6000/rs6000-builtins.def b/gcc/config/rs6000/rs6000-builtins.def index 513c69e48b8..d30dc5ae0f6 100644 --- a/gcc/config/rs6000/rs6000-builtins.def +++ b/gcc/config/rs6000/rs6000-builtins.def @@ -3980,3 +3980,88 @@ const int<8>, const int<4>, \ const int<4>); PMDMXVI8GERX4PP_INTERNAL dmf_pmdmxvi8gerx4pp {dm,pair} + + void __builtin_mma_dmxvi8gerx4spp (dm1024 *, v256, vuc); + DMXVI8GERX4SPP nothing {dm,dmint,dmr} + + dm1024 __builtin_mma_dmxvi8gerx4spp_internal (dm1024, v256, vuc); + DMXVI8GERX4SPP_INTERNAL dmf_dmxvi8gerx4spp {dm} + + void __builtin_mma_pmdmxvi8gerx4spp (dm1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<4>); + PMDMXVI8GERX4SPP nothing {dm,pair,dmint,dmr} + + dm1024 __builtin_mma_pmdmxvi8gerx4spp_internal (dm1024, v256, vuc, \ + const int<8>, const int<4>, \ + const int<4>); + PMDMXVI8GERX4SPP_INTERNAL dmf_pmdmxvi8gerx4spp {dm,pair} + + void __builtin_mma_dmxvbf16gerx2 (dm1024 *, v256, vuc); + DMXVBF16GERX2 nothing {dm,dmint} + + dm1024 __builtin_mma_dmxvbf16gerx2_internal (v256, vuc); + DMXVBF16GERX2_INTERNAL dmf_dmxvbf16gerx2 {dm} + + void __builtin_mma_dmxvbf16gerx2pp (dm1024 *, v256, vuc); + DMXVBF16GERX2PP nothing {dm,dmint,dmr} + + dm1024 __builtin_mma_dmxvbf16gerx2pp_internal (dm1024, v256, vuc); + DMXVBF16GERX2PP_INTERNAL dmf_dmxvbf16gerx2pp {dm} + + void __builtin_mma_dmxvbf16gerx2pn (dm1024 *, v256, vuc); + DMXVBF16GERX2PN nothing {dm,dmint,dmr} + + dm1024 __builtin_mma_dmxvbf16gerx2pn_internal (dm1024, v256, vuc); + DMXVBF16GERX2PN_INTERNAL dmf_dmxvbf16gerx2pn {dm} + + void __builtin_mma_dmxvbf16gerx2np (dm1024 *, v256, vuc); + DMXVBF16GERX2NP nothing {dm,dmint,dmr} + + dm1024 __builtin_mma_dmxvbf16gerx2np_internal (dm1024, v256, vuc); + DMXVBF16GERX2NP_INTERNAL dmf_dmxvbf16gerx2np {dm} + + void __builtin_mma_dmxvbf16gerx2nn (dm1024 *, v256, vuc); + DMXVBF16GERX2NN nothing {dm,dmint,dmr} + + dm1024 __builtin_mma_dmxvbf16gerx2nn_internal (dm1024, v256, vuc); + DMXVBF16GERX2NN_INTERNAL dmf_dmxvbf16gerx2nn {dm} + + void __builtin_mma_pmdmxvbf16gerx2 (dm1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2 nothing {dm,pair,dmint} + + dm1024 __builtin_mma_pmdmxvbf16gerx2_internal (v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2_INTERNAL dmf_pmdmxvbf16gerx2 {dm,pair} + + void __builtin_mma_pmdmxvbf16gerx2pp (dm1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2PP nothing {dm,pair,dmint,dmr} + + dm1024 __builtin_mma_pmdmxvbf16gerx2pp_internal (dm1024, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2PP_INTERNAL dmf_pmdmxvbf16gerx2pp {dm,pair} + + void __builtin_mma_pmdmxvbf16gerx2pn (dm1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2PN nothing {dm,pair,dmint,dmr} + + dm1024 __builtin_mma_pmdmxvbf16gerx2pn_internal (dm1024, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2PN_INTERNAL dmf_pmdmxvbf16gerx2pn {dm,pair} + + void __builtin_mma_pmdmxvbf16gerx2np (dm1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2NP nothing {dm,pair,dmint,dmr} + + dm1024 __builtin_mma_pmdmxvbf16gerx2np_internal (dm1024, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2NP_INTERNAL dmf_pmdmxvbf16gerx2np {dm,pair} + + void __builtin_mma_pmdmxvbf16gerx2nn (dm1024 *, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2NN nothing {dm,pair,dmint,dmr} + + dm1024 __builtin_mma_pmdmxvbf16gerx2nn_internal (dm1024, v256, vuc, const int<8>, \ + const int<4>, const int<2>); + PMDMXVBF16GERX2NN_INTERNAL dmf_pmdmxvbf16gerx2nn {dm,pair} diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi index 7e438940c5b..4733f7c4947 100644 --- a/gcc/doc/extend.texi +++ b/gcc/doc/extend.texi @@ -27110,9 +27110,23 @@ The built-in functions supported are: @smallexample void __builtin_mma_dmxvi8gerx4 (__dm1024 *, __vector_pair, vec_t); void __builtin_mma_dmxvi8gerx4pp (__dm1024 *, __vector_pair, vec_t); +void __builtin_mma_dmxvi8gerx4spp (__dm1024 *, __vector_pair, vec_t); void __builtin_mma_pmdmxvi8gerx4 (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint4); void __builtin_mma_pmdmxvi8gerx4pp (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint4); +void __builtin_mma_pmdmxvi8gerx4spp (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint4); + +void __builtin_mma_dmxvbf16gerx2 (__dm1024 *, __vector_pair, vec_t); +void __builtin_mma_dmxvbf16gerx2pp (__dm1024 *, __vector_pair, vec_t); +void __builtin_mma_dmxvbf16gerx2pn (__dm1024 *, __vector_pair, vec_t); +void __builtin_mma_dmxvbf16gerx2np (__dm1024 *, __vector_pair, vec_t); +void __builtin_mma_dmxvbf16gerx2nn (__dm1024 *, __vector_pair, vec_t); + +void __builtin_mma_pmdmxvbf16gerx2 (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint2); +void __builtin_mma_pmdmxvbf16gerx2pp (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint2); +void __builtin_mma_pmdmxvbf16gerx2pn (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint2); +void __builtin_mma_pmdmxvbf16gerx2np (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint2); +void __builtin_mma_pmdmxvbf16gerx2nn (__dm1024 *, __vector_pair, vec_t, uint8, uint4, uint2); @end smallexample @node PowerPC Dense Math Facility Built-in Functions diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-builtin-1.c b/gcc/testsuite/gcc.target/powerpc/dmf-builtin-1.c new file mode 100644 index 00000000000..77d4818194d --- /dev/null +++ b/gcc/testsuite/gcc.target/powerpc/dmf-builtin-1.c @@ -0,0 +1,204 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target powerpc_future_compile_ok } */ +/* { dg-require-effective-target lp64 } */ +/* { dg-options "-mdejagnu-cpu=future -O2" } */ + +typedef unsigned char vec_t __attribute__((vector_size(16))); + +void +foo (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvbf16gerx2 (&dmr, vp, vec); + *dst = dmr; +} + +void +bar (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr = dst[0];; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvbf16gerx2 (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvbf16gerx2\M} 2 } } */ + +void +foo_1 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvbf16gerx2nn (&dmr, vp, vec); + *dst = dmr; +} + +void +bar_1 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr = dst[0];; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvbf16gerx2nn (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvbf16gerx2nn\M} 2 } } */ + +void +foo_2 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvbf16gerx2np (&dmr, vp, vec); + *dst = dmr; +} + +void +bar_2 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr = dst[0];; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvbf16gerx2np (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvbf16gerx2np\M} 2 } } */ + +void +foo_3 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvbf16gerx2pn (&dmr, vp, vec); + *dst = dmr; +} + +void +bar_3 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr = dst[0];; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvbf16gerx2pn (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvbf16gerx2pn\M} 2 } } */ + +void +foo_4 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvbf16gerx2pp (&dmr, vp, vec); + *dst = dmr; +} + +void +bar_4 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr = dst[0];; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvbf16gerx2pp (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvbf16gerx2pp\M} 2 } } */ + +void +foo_5 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvbf16gerx2 (dst, vp, vec, 255, 15, 2); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvbf16gerx2\M} 1 } } */ + +void +foo_6 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvbf16gerx2nn (dst, vp, vec, 255, 15, 2); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvbf16gerx2nn\M} 1 } } */ + +void +foo_7 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvbf16gerx2np (dst, vp, vec, 255, 15, 2); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvbf16gerx2np\M} 1 } } */ + +void +foo_8 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvbf16gerx2pn (dst, vp, vec, 255, 15, 2); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvbf16gerx2pn\M} 1 } } */ + +void +foo_9 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvbf16gerx2pp (dst, vp, vec, 255, 15, 2); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvbf16gerx2pp\M} 1 } } */ + +void +foo_10 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_pmdmxvi8gerx4spp (dst, vp, vec, 255, 15, 15); +} + +/* { dg-final { scan-assembler-times {\mpmdmxvi8gerx4spp\M} 1 } } */ + +void +foo_11 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_dmsetdmrz (&dmr); + __builtin_mma_dmxvi8gerx4spp (&dmr, vp, vec); + *dst = dmr; +} + +void +bar_11 (__dm1024 *dst, __vector_pair *vpp, vec_t *src) +{ + __dm1024 dmr = dst[0];; + __vector_pair vp = *vpp; + vec_t vec = *src; + __builtin_mma_dmxvi8gerx4spp (&dmr, vp, vec); + dst[1] = dmr; +} + +/* { dg-final { scan-assembler-times {\mdmxvi8gerx4spp\M} 2 } } */ -- 2.52.0
