Hello,
Below patch is bootstrapped and regtested on Power64le-linux-gnu.
Changes from V1:
- Updated testcase ChangeLog.
- Rename UNSPEC_DM_* unspec to UNSPEC_DMF_*.
- Update builtin documentation to use the __dmr1024 instead of
__dm1024 type.
Thank You,
Kishan
Add support for the DMF (Dense Math Facility) and MMA+
(Matrix-Multiply Assist Plus) builtins and instructions which may be
available for future Power processor.
This patch extends the existing MMA infrastructure to support Dense
Math Registers (DMRs).
Key changes:
1. Extended MMA operand support from 7 to 9 operands (MAX_MMA_OPERANDS)
2. Added new DMF-specific unspecs for DMR operations:
- UNSPEC_DMF_INSERT1024 for 1024-bit DMR insertions
- UNSPEC_DMF_DMXOR for DMR XOR operations
- UNSPEC_DMF_DMXVI8GERX4* for DMR GER (outer product) operations
- UNSPEC_DMF_PMDMXVI8GERX4* for prefixed DMR GER operations
- UNSPEC_DMF_INSERT512 for 512-bit DMR insertions
- UNSPEC_DMF_DMSETDMRZ for zeroing DMR register
- UNSPEC_DMF_RELOAD_FROM_MEMORY to restore a 1024-bit DMR from memory
3. Implemented new instruction patterns in mma.md:
- dm_insert512: Insert 512-bit data into DMR
- dm_insert1024: Insert 1024-bit data into DMR
- dmf_dmxor: XOR operation on DMR registers
- dmf_dmxvi8gerx4/dmxvi8gerx4pp: DMR outer product operations
- dmf_pmdmxvi8gerx4/pmdmxvi8gerx4pp: Prefixed DMR outer product
operations
4. Added new predicate for DMR register handling:
- dmr_register_operand: Validates DMR register operands
5. Added new builtins and Updated gimple folding:
- Added new DMF builtins (DMSETDMRZ, DMXOR, BUILD_DMR, etc.)
- Updated gimple folding to handle DMR pass-by-reference semantics
- Extended builtin expansion to support 8 and 9 operand instructions
6. Added ISA support:
- New "dmf" ISA attribute for Dense Math instructions
- New "dmf" and "mma" instruction type for scheduling
The implementation follows the existing MMA pattern where user-facing
builtins use pass-by-reference for accumulator/DMR arguments, while
internal builtins use pass-by-value for optimization.
This patch depends on the following prerequisite patches:
rs6000: Add support for Dense Math Facility (DMF) registers
https://patchwork.ozlabs.org/project/gcc/patch/
[email protected]/
rs6000: Add the __dmr1024 opaque type
https://patchwork.ozlabs.org/project/gcc/patch/
[email protected]/
rs6000: Add %wD constraint and predicate for accumulators
https://patchwork.ozlabs.org/project/gcc/patch/
[email protected]/
[PATCH v3] rs6000: Add Dense Math builtin infrastructure
https://patchwork.ozlabs.org/project/gcc/patch/
[email protected]/
2026-07-13 Peter Bergner <[email protected]>
Surya Kumari jangala <[email protected]>
Kishan Parmar <[email protected]>
gcc/ChangeLog:
* config/rs6000/mma.md (MAX_MMA_OPERANDS): Increase from 7 to 9.
(UNSPEC_DMF_INSERT512): New unspec.
(UNSPEC_DMF_INSERT1024): Likewise.
(UNSPEC_DMF_DMXOR): Likewise.
(UNSPEC_DMF_DMXVI8GERX4): Likewise.
(UNSPEC_DMF_DMXVI8GERX4PP): Likewise.
(UNSPEC_DMF_PMDMXVI8GERX4): Likewise.
(UNSPEC_DMF_PMDMXVI8GERX4PP): Likewise.
(UNSPEC_DMF_DMSETDMRZ): Likewise.
(UNSPEC_DMF_RELOAD_FROM_MEMORY): Likewise.
(DMF_PV): New int iterator.
(DMF_DPV): Likewise.
(DMF_PVI8I4I4): Likewise.
(DMF_DPVI8I4I4): Likewise.
(pv): Add DMF attribute.
(apv): Likewise.
(pvi8i4i4): New attribute.
(dpvi8i4i4): Likewise.
(dm_insert512): New insn.
(dm_insert1024): Likewise.
(dmf_build_dmr): New define_expand.
(dmf_dmsetdmrz): New insn.
(dmf_dmxor): Likewise.
(reload_tdo_from_memory): New insn_and_split.
(dmf_<pv>): New insn pattern.
(dmf_<apv>): Likewise.
(dmf_<pvi8i4i4>): Likewise.
(dmf_<dpvi8i4i4>): Likewise.
* config/rs6000/predicates.md (dmr_register_operand): New predicate.
(gpc_reg_operand): Accept DMR registers.
* config/rs6000/rs6000-builtin.cc
(rs6000_gimple_fold_mma_builtin): Add support for DMF builtins.
Handle DMR pass-by-reference semantics.
Support 8 and 9 operand builtins.
(mma_expand_builtin): Support 8 and 9 operand builtins.
(rs6000_expand_builtin): Add DMF builtin support.
Increase MAX_BUILTIN_ARGS from 6 to 8.
* config/rs6000/rs6000-builtins.def: Add DMF builtin definitions.
* config/rs6000/rs6000.md (type): Add dmf type.
(isa): Add mma and dmf ISA attributes.
(enabled): Add mma and dmf enable conditions.
* doc/extend.texi (PowerPC Dense Math Facility Built-in Functions):
Document DMF builtins.
(PowerPC Matrix-Multiply Assist Built-in Functions): Document
MMA+ builtins.
gcc/testsuite/ChangeLog:
* gcc.target/powerpc/dmf-build-dmr.c: New test.
* gcc.target/powerpc/dmf-builtin.c: New test
---
gcc/config/rs6000/mma.md | 198 +++++++++++++++++-
gcc/config/rs6000/predicates.md | 19 ++
gcc/config/rs6000/rs6000-builtin.cc | 81 +++++--
gcc/config/rs6000/rs6000-builtins.def | 49 +++++
gcc/config/rs6000/rs6000.md | 12 +-
gcc/doc/extend.texi | 38 ++++
.../gcc.target/powerpc/dmf-build-dmr.c | 15 ++
.../gcc.target/powerpc/dmf-builtin.c | 81 +++++++
8 files changed, 462 insertions(+), 31 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c
create mode 100644 gcc/testsuite/gcc.target/powerpc/dmf-builtin.c
diff --git a/gcc/config/rs6000/mma.md b/gcc/config/rs6000/mma.md
index 1103f1fc037..7f0c26c6eb5 100644
--- a/gcc/config/rs6000/mma.md
+++ b/gcc/config/rs6000/mma.md
@@ -24,7 +24,7 @@
;; __vector_pair types that the MMA built-in functions reference. We
;; use OPAQUE_MODE to prevent anything from trying to open them up.
-(define_constants [(MAX_MMA_OPERANDS 7)])
+(define_constants [(MAX_MMA_OPERANDS 9)])
;; Constants for creating unspecs
@@ -91,6 +91,15 @@ (define_c_enum "unspec"
UNSPEC_MMA_XVI8GER4SPP
UNSPEC_MMA_XXMFACC
UNSPEC_MMA_XXMTACC
+ UNSPEC_DMF_INSERT512
+ UNSPEC_DMF_INSERT1024
+ UNSPEC_DMF_DMXOR
+ UNSPEC_DMF_DMXVI8GERX4
+ UNSPEC_DMF_DMXVI8GERX4PP
+ UNSPEC_DMF_PMDMXVI8GERX4
+ UNSPEC_DMF_PMDMXVI8GERX4PP
+ UNSPEC_DMF_DMSETDMRZ
+ UNSPEC_DMF_RELOAD_FROM_MEMORY
])
(define_c_enum "unspecv"
@@ -133,12 +142,18 @@ (define_int_iterator MMA_AVV
[UNSPEC_MMA_XVI4GER8PP
;; MMA instructions with 1 vector pair and 1 vector arguments
(define_int_iterator MMA_PV [UNSPEC_MMA_XVF64GER])
+; DMF instructions with 1 vector pair and 1 vector arguments
+(define_int_iterator DMF_PV [UNSPEC_DMF_DMXVI8GERX4])
+
;; MMA instructions with 1 accumulator, 1 vector pair and 1 vector arguments
(define_int_iterator MMA_APV [UNSPEC_MMA_XVF64GERPP
UNSPEC_MMA_XVF64GERPN
UNSPEC_MMA_XVF64GERNP
UNSPEC_MMA_XVF64GERNN])
+;; DMF instructions with 1 dmr, 1 vector pair and 1 vector arguments
+(define_int_iterator DMF_DPV [UNSPEC_DMF_DMXVI8GERX4PP])
+
;; MMA instructions with 2 vector, 2 4-bit and 1 8-bit arguments
(define_int_iterator MMA_VVI4I4I8 [UNSPEC_MMA_PMXVI4GER8])
@@ -188,6 +203,14 @@ (define_int_iterator MMA_VVI4I4I4 [UNSPEC_MMA_PMXVI8GER4])
(define_int_iterator MMA_AVVI4I4I4 [UNSPEC_MMA_PMXVI8GER4PP
UNSPEC_MMA_PMXVI8GER4SPP])
+; DMF instructions with 1 vector pair, 1 vector and 1 8-bit and 2 4-bit
+;; arguments
+(define_int_iterator DMF_PVI8I4I4 [UNSPEC_DMF_PMDMXVI8GERX4])
+
+;; DMF instructions with 1dmr, 1 vector pair, 1 vector and 1 8-bit and
+;; 2 4-bit arguments
+(define_int_iterator DMF_DPVI8I4I4 [UNSPEC_DMF_PMDMXVI8GERX4PP])
+
(define_int_attr acc [(UNSPEC_MMA_XXMFACC "xxmfacc")
(UNSPEC_MMA_XXMTACC "xxmtacc")])
@@ -217,12 +240,14 @@ (define_int_attr avv [(UNSPEC_MMA_XVI4GER8PP
"xvi4ger8pp")
(UNSPEC_MMA_XVF32GERNP "xvf32gernp")
(UNSPEC_MMA_XVF32GERNN "xvf32gernn")])
-(define_int_attr pv [(UNSPEC_MMA_XVF64GER "xvf64ger")])
+(define_int_attr pv [(UNSPEC_MMA_XVF64GER "xvf64ger")
+ (UNSPEC_DMF_DMXVI8GERX4 "dmxvi8gerx4")])
(define_int_attr apv [(UNSPEC_MMA_XVF64GERPP "xvf64gerpp")
(UNSPEC_MMA_XVF64GERPN "xvf64gerpn")
(UNSPEC_MMA_XVF64GERNP "xvf64gernp")
- (UNSPEC_MMA_XVF64GERNN "xvf64gernn")])
+ (UNSPEC_MMA_XVF64GERNN "xvf64gernn")
+ (UNSPEC_DMF_DMXVI8GERX4PP
"dmxvi8gerx4pp")])
(define_int_attr vvi4i4i8 [(UNSPEC_MMA_PMXVI4GER8 "pmxvi4ger8")])
@@ -263,6 +288,9 @@ (define_int_attr vvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4
"pmxvi8ger4")])
(define_int_attr avvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4PP "pmxvi8ger4pp")
(UNSPEC_MMA_PMXVI8GER4SPP
"pmxvi8ger4spp")])
+(define_int_attr pvi8i4i4 [(UNSPEC_DMF_PMDMXVI8GERX4
"pmdmxvi8gerx4")])
+
+(define_int_attr dpvi8i4i4 [(UNSPEC_DMF_PMDMXVI8GERX4PP
"pmdmxvi8gerx4pp")])
;; Vector pair support. OOmode can only live in VSRs.
(define_expand "movoo"
@@ -425,6 +453,29 @@ (define_insn_and_split "*vsx_disassemble_pair"
DONE;
})
+(define_insn "dm_insert512"
+ [(set (match_operand:XO 0 "dmr_register_operand" "=wD")
+ (unspec:XO [(match_operand:OO 1 "vsx_register_operand" "wa")
+ (match_operand:OO 2 "vsx_register_operand" "wa")
+ (match_operand 3 "const_0_to_1_operand")]
+ UNSPEC_DMF_INSERT512))]
+ "TARGET_DMF"
+ "dmxxinstdmr512 %0,%x1,%x2,%3"
+ [(set_attr "type" "dmf")])
+
+;; Move from VSX registers to DMR registers via two insert 512 bit
+;; instructions.
+(define_insn "dm_insert1024"
+ [(set (match_operand:TDO 0 "dmr_register_operand" "=wD")
+ (unspec:TDO [(match_operand:OO 1 "vsx_register_operand" "wa")
+ (match_operand:OO 2 "vsx_register_operand" "wa")
+ (match_operand:OO 3 "vsx_register_operand" "wa")
+ (match_operand:OO 4 "vsx_register_operand" "wa")]
+ UNSPEC_DMF_INSERT1024))]
+ "TARGET_DMF"
+ "dmxxinstdmr512 %0,%x1,%x2,0\n\tdmxxinstdmr512 %0,%x3,%x4,1"
+ [(set_attr "type" "dmf")])
+
(define_expand "mma_assemble_acc"
[(match_operand:XO 0 "fpr_reg_operand")
(match_operand:V16QI 1 "mma_assemble_input_operand")
@@ -434,9 +485,9 @@ (define_expand "mma_assemble_acc"
"TARGET_MMA"
{
rtx src = gen_rtx_UNSPEC_VOLATILE (XOmode,
- gen_rtvec (4, operands[1], operands[2],
- operands[3], operands[4]),
- UNSPECV_MMA_ASSEMBLE);
+ gen_rtvec (4, operands[1], operands[2],
+ operands[3], operands[4]),
+ UNSPECV_MMA_ASSEMBLE);
emit_move_insn (operands[0], src);
DONE;
})
@@ -466,6 +517,30 @@ (define_insn_and_split "*mma_assemble_acc"
DONE;
})
+(define_expand "dmf_build_dmr"
+ [(match_operand:TDO 0 "dmr_register_operand")
+ (match_operand:V16QI 1 "mma_assemble_input_operand")
+ (match_operand:V16QI 2 "mma_assemble_input_operand")
+ (match_operand:V16QI 3 "mma_assemble_input_operand")
+ (match_operand:V16QI 4 "mma_assemble_input_operand")
+ (match_operand:V16QI 5 "mma_assemble_input_operand")
+ (match_operand:V16QI 6 "mma_assemble_input_operand")
+ (match_operand:V16QI 7 "mma_assemble_input_operand")
+ (match_operand:V16QI 8 "mma_assemble_input_operand")]
+ "TARGET_DMF"
+{
+ rtx vp0 = gen_reg_rtx (OOmode);
+ rtx vp1 = gen_reg_rtx (OOmode);
+ rtx vp2 = gen_reg_rtx (OOmode);
+ rtx vp3 = gen_reg_rtx (OOmode);
+ emit_insn (gen_vsx_assemble_pair (vp0, operands[2], operands[1]));
+ emit_insn (gen_vsx_assemble_pair (vp1, operands[4], operands[3]));
+ emit_insn (gen_vsx_assemble_pair (vp2, operands[6], operands[5]));
+ emit_insn (gen_vsx_assemble_pair (vp3, operands[8], operands[7]));
+ emit_insn (gen_dm_insert1024 (operands[0], vp0, vp1, vp2, vp3));
+ DONE;
+})
+
(define_expand "mma_disassemble_acc"
[(match_operand:V16QI 0 "mma_disassemble_output_operand")
(match_operand:XO 1 "fpr_reg_operand")
@@ -522,6 +597,13 @@ (define_insn "mma_xxsetaccz"
"xxsetaccz %A0"
[(set_attr "type" "mma")])
+(define_insn "dmf_dmsetdmrz"
+ [(set (match_operand:TDO 0 "dmr_register_operand" "=wD")
+ (unspec:TDO [(const_int 0)] UNSPEC_DMF_DMSETDMRZ))]
+ "TARGET_DMF"
+ "dmsetdmrz %0"
+ [(set_attr "type" "dmf")])
+
(define_insn "mma_<vv>"
[(set (match_operand:XO 0 "fpr_reg_operand" "=&d,&d")
(unspec:XO [(match_operand:V16QI 1 "vsx_register_operand" "v,?wa")
@@ -690,3 +772,107 @@ (define_insn "mma_<avvi4i4i4>"
"<avvi4i4i4> %A0,%x2,%x3,%4,%5,%6"
[(set_attr "type" "mma")
(set_attr "prefixed" "yes")])
+
+(define_insn "dmf_dmxor"
+ [(set (match_operand:TDO 0 "dmr_register_operand" "=wD")
+ (unspec:TDO [(match_operand:TDO 1 "dmr_register_operand" "0")
+ (match_operand:TDO 2 "dmr_register_operand" "wD")]
+ UNSPEC_DMF_DMXOR))]
+ "TARGET_DMF"
+ "dmxor %0,%2"
+ [(set_attr "type" "dmf")])
+
+;; Reload dense math registers from memory.
+(define_insn_and_split "reload_tdo_from_memory"
+ [(set (match_operand:TDO 0 "dmr_register_operand" "=wD")
+ (unspec:TDO [(match_operand:TDO 1 "memory_operand" "m")]
+ UNSPEC_DMF_RELOAD_FROM_MEMORY))
+ (clobber (match_operand:XO 2 "vsx_register_operand" "=wa"))]
+ "TARGET_DMF"
+ "#"
+ "&& reload_completed"
+ [(const_int 0)]
+{
+ rtx dest = operands[0];
+ rtx src = operands[1];
+ rtx pair0 = operands[2];
+ rtx pair1 = operands[3];
+ rtx pair2 = operands[4];
+ rtx pair3 = operands[5];
+
+ if (BYTES_BIG_ENDIAN)
+ {
+ emit_move_insn (pair0, adjust_address (src, OOmode, 0));
+ emit_move_insn (pair1, adjust_address (src, OOmode, 32));
+ emit_move_insn (pair2, adjust_address (src, OOmode, 64));
+ emit_move_insn (pair3, adjust_address (src, OOmode, 96));
+ }
+ else
+ {
+ emit_move_insn (pair3, adjust_address (src, OOmode, 0));
+ emit_move_insn (pair2, adjust_address (src, OOmode, 32));
+ emit_move_insn (pair1, adjust_address (src, OOmode, 64));
+ emit_move_insn (pair0, adjust_address (src, OOmode, 96));
+ }
+
+ emit_insn (gen_dm_insert1024 (dest, pair0, pair1, pair2, pair3));
+ DONE;
+}
+ [(set_attr "length" "20")
+ (set_attr "max_prefixed_insns" "2")
+ (set_attr "type" "vecload")])
+
+(define_insn "dmf_<pv>"
+ [(set (match_operand:TDO 0 "accumulator_operand" "=wD")
+ (unspec:TDO [(match_operand:OO 1 "vsx_register_operand" "wa")
+ (match_operand:V16QI 2 "vsx_register_operand" "wa")]
+ DMF_PV))]
+ "TARGET_DMF"
+{
+ return "<pv> %0,%x1,%x2";
+}
+ [(set_attr "type" "dmf")])
+
+(define_insn "dmf_<apv>"
+ [(set (match_operand:TDO 0 "accumulator_operand" "=wD")
+ (unspec:TDO [(match_operand:TDO 1 "accumulator_operand" "0")
+ (match_operand:OO 2 "vsx_register_operand" "wa")
+ (match_operand:V16QI 3 "vsx_register_operand" "wa")]
+ DMF_DPV))]
+ "TARGET_DMF"
+{
+ return "<apv> %0,%x2,%x3";
+}
+ [(set_attr "type" "dmf")])
+
+(define_insn "dmf_<pvi8i4i4>"
+ [(set (match_operand:TDO 0 "dmr_register_operand" "=wD")
+ (unspec:TDO [(match_operand:OO 1 "vsx_register_operand" "wa")
+ (match_operand:V16QI 2 "vsx_register_operand" "wa")
+ (match_operand:SI 3 "u8bit_cint_operand" "n")
+ (match_operand:SI 4 "const_0_to_15_operand" "n")
+ (match_operand:SI 5 "const_0_to_15_operand" "n")]
+ DMF_PVI8I4I4))]
+ "TARGET_DMF"
+{
+ return "<pvi8i4i4> %0,%x1,%x2,%3,%4,%5";
+}
+ [(set_attr "type" "dmf")
+ (set_attr "prefixed" "yes")])
+
+(define_insn "dmf_<dpvi8i4i4>"
+ [(set (match_operand:TDO 0 "dmr_register_operand" "=wD")
+ (unspec:TDO [(match_operand:TDO 1 "dmr_register_operand" "0")
+ (match_operand:OO 2 "vsx_register_operand" "wa")
+ (match_operand:V16QI 3 "vsx_register_operand" "wa")
+ (match_operand:SI 4 "u8bit_cint_operand" "n")
+ (match_operand:SI 5 "const_0_to_15_operand" "n")
+ (match_operand:SI 6 "const_0_to_15_operand" "n")]
+ DMF_DPVI8I4I4))]
+ "TARGET_DMF"
+{
+ return "<dpvi8i4i4> %0,%x2,%x3,%4,%5,%6";
+}
+ [(set_attr "type" "dmf")
+ (set_attr "prefixed" "yes")])
+
diff --git a/gcc/config/rs6000/predicates.md b/gcc/config/rs6000/predicates.md
index 9d4928b0464..a6ec04f5922 100644
--- a/gcc/config/rs6000/predicates.md
+++ b/gcc/config/rs6000/predicates.md
@@ -163,6 +163,22 @@ (define_predicate "vint_operand"
return VINT_REGNO_P (REGNO (op));
})
+;; Return 1 if op is a dense math register
+(define_predicate "dmr_register_operand"
+ (match_operand 0 "register_operand")
+{
+ if (!TARGET_DMF)
+ return 0;
+
+ if (!REG_P (op))
+ return 0;
+
+ if (!HARD_REGISTER_P (op))
+ return 1;
+
+ return DMR_REGNO_P (REGNO (op));
+})
+
;; Return 1 if op is an accumulator. On power10/11 systems, the accumulators
;; overlap with the FPRs. If TARGET_DMF is true, it will be Dense math
register.
(define_predicate "accumulator_operand"
@@ -387,6 +403,9 @@ (define_predicate "gpc_reg_operand"
if (TARGET_VSX && VSX_REGNO_P (REGNO (op)))
return 1;
+ if (TARGET_DMF && DMR_REGNO_P (REGNO (op)))
+ return 1;
+
return INT_REGNO_P (REGNO (op)) || FP_REGNO_P (REGNO (op));
})
diff --git a/gcc/config/rs6000/rs6000-builtin.cc
b/gcc/config/rs6000/rs6000-builtin.cc
index cea668fb163..95372aaac71 100644
--- a/gcc/config/rs6000/rs6000-builtin.cc
+++ b/gcc/config/rs6000/rs6000-builtin.cc
@@ -1121,7 +1121,8 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi,
gimple *stmt = gsi_stmt (*gsi);
size_t fncode = (size_t) fn_code;
- if (!bif_is_mma (rs6000_builtin_info[fncode]))
+ if (!bif_is_mma (rs6000_builtin_info[fncode])
+ && !bif_is_dm (rs6000_builtin_info[fncode]))
return false;
/* Each call that can be gimple-expanded has an associated built-in
@@ -1133,7 +1134,6 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi,
return false;
bifdata *bd = &rs6000_builtin_info[fncode];
- unsigned nopnds = bd->nargs;
gimple_seq new_seq = NULL;
gimple *new_call;
tree new_decl;
@@ -1249,27 +1249,49 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator
*gsi,
/* Convert this built-in into an internal version that uses pass-by-value
arguments. The internal built-in is found in the assoc_bif field. */
- new_decl = rs6000_builtin_decls[rs6000_builtin_info[fncode].assoc_bif];
+ size_t new_fncode = rs6000_builtin_info[fncode].assoc_bif;
+ new_decl = rs6000_builtin_decls[new_fncode];
tree lhs, op[MAX_MMA_OPERANDS];
+ tree lhs_type = NULL_TREE;
tree acc = gimple_call_arg (stmt, 0);
push_gimplify_context (true);
- if (bif_is_quad (*bd))
+ switch (insn_data[rs6000_builtin_info[new_fncode].icode].operand[0].mode)
{
- /* This built-in has a pass-by-reference accumulator input, so load it
- into a temporary accumulator for use as a pass-by-value input. */
- op[0] = make_ssa_name (vector_quad_type_node);
- for (unsigned i = 1; i < nopnds; i++)
- op[i] = gimple_call_arg (stmt, i);
- gimplify_assign (op[0], build_simple_mem_ref (acc), &new_seq);
+ case TDOmode:
+ lhs_type = dmr1024_type_node;
+ break;
+ case XOmode:
+ lhs_type = vector_quad_type_node;
+ break;
+ case OOmode:
+ lhs_type = vector_pair_type_node;
+ break;
+ default:
+ gcc_unreachable ();
}
- else
- {
- /* This built-in does not use its pass-by-reference accumulator argument
- as an input argument, so remove it from the input list. */
- nopnds--;
- for (unsigned i = 0; i < nopnds; i++)
- op[i] = gimple_call_arg (stmt, i + 1);
+
+ unsigned nopnds = 0;
+ for (int i = 0; i < bd->nargs; i++)
+ {
+ tree arg = gimple_call_arg (stmt, i);
+ if (i == 0 && !bif_is_dmr (*bd) && !bif_is_quad (*bd))
+ continue;
+ /* If this is another DMR operand, it is passed in by reference.
+ The internal built-ins use pass-by-value, so load this operand
+ into a variable and pass that in as our operand. */
+ if (POINTER_TYPE_P (TREE_TYPE (arg))
+ && types_compatible_p (TREE_TYPE (TREE_TYPE (arg)), lhs_type))
+ {
+ tree op_mem = build_simple_mem_ref (build1 (NOP_EXPR,
+ TREE_TYPE (arg),
+ arg));
+ op[nopnds] = make_ssa_name (lhs_type);
+ gimplify_assign (op[nopnds], op_mem, &new_seq);
+ }
+ else
+ op[nopnds] = arg;
+ nopnds++;
}
switch (nopnds)
@@ -1301,14 +1323,19 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator
*gsi,
new_call = gimple_build_call (new_decl, 7, op[0], op[1], op[2], op[3],
op[4], op[5], op[6]);
break;
+ case 8:
+ new_call = gimple_build_call (new_decl, 8, op[0], op[1], op[2], op[3],
+ op[4], op[5], op[6], op[7]);
+ break;
+ case 9:
+ new_call = gimple_build_call (new_decl, 9, op[0], op[1], op[2], op[3],
+ op[4], op[5], op[6], op[7], op[8]);
+ break;
default:
gcc_unreachable ();
}
- if (fncode == RS6000_BIF_BUILD_PAIR || fncode == RS6000_BIF_ASSEMBLE_PAIR_V)
- lhs = make_ssa_name (vector_pair_type_node);
- else
- lhs = make_ssa_name (vector_quad_type_node);
+ lhs = make_ssa_name (lhs_type);
gimple_call_set_lhs (new_call, lhs);
gimple_seq_add_stmt (&new_seq, new_call);
gimplify_assign (build_simple_mem_ref (acc), lhs, &new_seq);
@@ -3025,6 +3052,14 @@ mma_expand_builtin (tree exp, rtx target, insn_code
icode,
case 7:
pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5], op[6]);
break;
+ case 8:
+ pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5], op[6],
+ op[7]);
+ break;
+ case 9:
+ pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5], op[6],
+ op[7], op[8]);
+ break;
default:
gcc_unreachable ();
}
@@ -3567,7 +3602,7 @@ rs6000_expand_builtin (tree exp, rtx target, rtx /*
subtarget */,
/* Position of first argument (0 for void-returning functions, else 1). */
int k;
/* Modes for the return value, if any, and arguments. */
- const int MAX_BUILTIN_ARGS = 6;
+ const int MAX_BUILTIN_ARGS = 8;
machine_mode mode[MAX_BUILTIN_ARGS + 1];
if (void_func)
@@ -3702,7 +3737,7 @@ rs6000_expand_builtin (tree exp, rtx target, rtx /*
subtarget */,
if (bif_is_lxvrze (*bifaddr))
return lxvrze_expand_builtin (target, icode, op, mode[0], mode[1]);
- if (bif_is_mma (*bifaddr))
+ if (bif_is_mma (*bifaddr) || bif_is_dm (*bifaddr))
return mma_expand_builtin (exp, target, icode, fcode);
if (TREE_TYPE (TREE_TYPE (fndecl)) == void_type_node)
diff --git a/gcc/config/rs6000/rs6000-builtins.def
b/gcc/config/rs6000/rs6000-builtins.def
index 85486c70f0d..9ffe92e9e12 100644
--- a/gcc/config/rs6000/rs6000-builtins.def
+++ b/gcc/config/rs6000/rs6000-builtins.def
@@ -4095,3 +4095,52 @@
const vf __builtin_altivec_unpack_int8_to_fp32 (vui, const int<2>);
VUPKINT8TOFP32 altivec_vupkint8tofp32 {}
+
+[dm]
+ void __builtin_dmsetdmrz (dmr1024 *);
+ DMSETDMRZ nothing {dm,dmint}
+
+ dmr1024 __builtin_dmsetdmrz_internal ();
+ DMSETDMRZ_INTERNAL dmf_dmsetdmrz {dm}
+
+ void __builtin_dmxor (dmr1024 *, dmr1024 *);
+ DMXOR nothing {dm,dmint,dmr}
+
+ dmr1024 __builtin_dmxor_internal (dmr1024, dmr1024);
+ DMXOR_INTERNAL dmf_dmxor {dm}
+
+ void __builtin_build_dmr (dmr1024 *, vuc, vuc, vuc, vuc, vuc, vuc, vuc, vuc);
+ BUILD_DMR nothing {dm,dmint}
+
+ dmr1024 __builtin_build_dmr_internal (vuc, vuc, vuc, vuc, vuc, vuc, vuc,
vuc);
+ BUILD_DMR_INTERNAL dmf_build_dmr {dm}
+
+ void __builtin_mma_dmxvi8gerx4 (dmr1024 *, v256, vuc);
+ DMXVI8GERX4 nothing {dm,dmint}
+
+ dmr1024 __builtin_mma_dmxvi8gerx4_internal (v256, vuc);
+ DMXVI8GERX4_INTERNAL dmf_dmxvi8gerx4 {dm}
+
+ void __builtin_mma_dmxvi8gerx4pp (dmr1024 *, v256, vuc);
+ DMXVI8GERX4PP nothing {dm,dmint,dmr}
+
+ dmr1024 __builtin_mma_dmxvi8gerx4pp_internal (dmr1024, v256, vuc);
+ DMXVI8GERX4PP_INTERNAL dmf_dmxvi8gerx4pp {dm}
+
+ void __builtin_mma_pmdmxvi8gerx4 (dmr1024 *, v256, vuc, const int<8>, \
+ const int<4>, const int<4>);
+ PMDMXVI8GERX4 nothing {dm,pair,dmint}
+
+ dmr1024 __builtin_mma_pmdmxvi8gerx4_internal (v256, vuc, const int<8>, \
+ const int<4>, const int<4>);
+ PMDMXVI8GERX4_INTERNAL dmf_pmdmxvi8gerx4 {dm,pair}
+
+ void __builtin_mma_pmdmxvi8gerx4pp (dmr1024 *, v256, vuc, const int<8>, \
+ const int<4>, const int<4>);
+ PMDMXVI8GERX4PP nothing {dm,pair,dmint,dmr}
+
+ dmr1024 __builtin_mma_pmdmxvi8gerx4pp_internal (dmr1024, v256, vuc, \
+ const int<8>, const int<4>, \
+ const int<4>);
+ PMDMXVI8GERX4PP_INTERNAL dmf_pmdmxvi8gerx4pp {dm,pair}
+
diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
index 86d79d92258..e06dcec6588 100644
--- a/gcc/config/rs6000/rs6000.md
+++ b/gcc/config/rs6000/rs6000.md
@@ -223,7 +223,7 @@ (define_attr "type"
vecsimple,veccomplex,vecdiv,veccmp,veccmpsimple,vecperm,
vecfloat,vecfdiv,vecdouble,mtvsr,mfvsr,crypto,
veclogical,veccmpfx,vecexts,vecmove,
- htm,htmsimple,dfp,mma,
+ htm,htmsimple,dfp,mma,dmf,
fused_arith_logical,
fused_cmp_isel,
fused_carry,
@@ -371,7 +371,7 @@ (define_attr "cpu"
(const (symbol_ref "(enum attr_cpu) rs6000_tune")))
;; The ISA we implement.
-(define_attr "isa" "any,p5,p6,p7,p7v,p8,p8v,p9,p9v,p9kf,p9tf,p10,future"
+(define_attr "isa"
"any,p5,p6,p7,p7v,p8,p8v,p9,p9v,p9kf,p9tf,p10,future,mma,dmf"
(const_string "any"))
;; Is this alternative enabled for the current CPU/ISA/etc.?
@@ -427,6 +427,14 @@ (define_attr "enabled" ""
(and (eq_attr "isa" "future")
(match_test "TARGET_FUTURE"))
(const_int 1)
+
+ (and (eq_attr "isa" "mma")
+ (match_test "TARGET_MMA"))
+ (const_int 1)
+
+ (and (eq_attr "isa" "dmf")
+ (match_test "TARGET_DMF"))
+ (const_int 1)
] (const_int 0)))
;; If this instruction is microcoded on the CELL processor
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index 60fc5add3bc..ff7cfdc85e6 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -18708,6 +18708,7 @@ instructions, but allow the compiler to schedule those
calls.
* PowerPC Hardware Transactional Memory Built-in Functions::
* PowerPC Atomic Memory Operation Functions::
* PowerPC Matrix-Multiply Assist Built-in Functions::
+* PowerPC Dense Math Facility Built-in Functions::
* PRU Built-in Functions::
* RISC-V Built-in Functions::
* RISC-V Vector Intrinsics::
@@ -27462,6 +27463,43 @@ __vector_pair __builtin_vsx_lxvp (size_t,
__vector_pair *);
void __builtin_vsx_stxvp (__vector_pair, size_t, __vector_pair *);
@end smallexample
+Future ISA of PowerPC may add new Matrix-Multiply Assist Plus(MMA+)
+instructions. GCC provides support for these instructions through the
+following built-in functions which are enabled with the @code{-mmma} option.
+The vec_t type below is defined to be a normal vector unsigned char type.
+The uint2, uint4 and uint8 parameters are 2-bit, 4-bit and 8-bit unsigned
+integer constants respectively. The compiler will verify that they are
+constants and that their values are within range. The __dmr1024 type is a
+1024 bit integer type.
+
+The built-in functions supported are:
+
+@smallexample
+void __builtin_mma_dmxvi8gerx4 (__dmr1024 *, __vector_pair, vec_t);
+void __builtin_mma_dmxvi8gerx4pp (__dmr1024 *, __vector_pair, vec_t);
+
+void __builtin_mma_pmdmxvi8gerx4 (__dmr1024 *, __vector_pair, vec_t, uint8,
uint4, uint4);
+void __builtin_mma_pmdmxvi8gerx4pp (__dmr1024 *, __vector_pair, vec_t, uint8,
uint4, uint4);
+@end smallexample
+
+@node PowerPC Dense Math Facility Built-in Functions
+@subsection PowerPC Dense Math Facility Built-in Functions
+
+A future PowerPC processor may provide Dense Math Facility (DMF)
+instructions. GCC provides support for these instructions through the
+following built-in functions which are enabled with the @code{-mdense-math}
+option. The vec_t type below is defined to be a normal vector unsigned char
+type. The __dmr1024 type is a 1024 bit integer type.
+
+The built-in functions supported are:
+
+@smallexample
+void __builtin_dmsetdmrz (__dmr1024 *);
+void __builtin_dmxor (__dmr1024 *, __dmr1024 *);
+void __builtin_build_dmr (__dmr1024 *, vec_t, vec_t, vec_t, vec_t, vec_t,
vec_t, vec_t, vec_t);
+@end smallexample
+
@node PRU Built-in Functions
@subsection PRU Built-in Functions
diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c
b/gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c
new file mode 100644
index 00000000000..d5e85e64e27
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/dmf-build-dmr.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_future_compile_ok } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-options "-mdejagnu-cpu=future -O2" } */
+
+typedef unsigned char vec_t __attribute__((vector_size(16)));
+
+void
+foo2 (__dmr1024 *dst, vec_t *src)
+{
+ __builtin_build_dmr (dst, src[0], src[1], src[2], src[3], src[4], src[5],
src[6], src[7]);
+}
+
+/* { dg-final { scan-assembler-times {\mdmxxinstdmr512\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mlxv\M} 8 } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/dmf-builtin.c
b/gcc/testsuite/gcc.target/powerpc/dmf-builtin.c
new file mode 100644
index 00000000000..434cf2ac2d3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/dmf-builtin.c
@@ -0,0 +1,81 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_future_compile_ok } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-options "-mdejagnu-cpu=future -O2" } */
+
+typedef unsigned char vec_t __attribute__((vector_size(16)));
+
+void
+foo (__dmr1024 *dst, __vector_pair *vpp, vec_t *src)
+{
+ __dmr1024 dmr;
+ __vector_pair vp = *vpp;
+ vec_t vec = *src;
+ __builtin_dmsetdmrz (&dmr);
+ __builtin_mma_dmxvi8gerx4 (&dmr, vp, vec);
+ *dst = dmr;
+}
+
+void
+bar (__dmr1024 *dst, __vector_pair *vpp, vec_t *src)
+{
+ __dmr1024 dmr = dst[0];;
+ __vector_pair vp = *vpp;
+ vec_t vec = *src;
+ __builtin_mma_dmxvi8gerx4 (&dmr, vp, vec);
+ dst[1] = dmr;
+}
+
+/* { dg-final { scan-assembler-times {\mdmxvi8gerx4\M} 2 } } */
+
+void
+foo_1 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src)
+{
+ __dmr1024 dmr;
+ __vector_pair vp = *vpp;
+ vec_t vec = *src;
+ __builtin_dmsetdmrz (&dmr);
+ __builtin_mma_dmxvi8gerx4pp (&dmr, vp, vec);
+ *dst = dmr;
+}
+
+void
+bar_1 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src)
+{
+ __dmr1024 dmr = dst[0];;
+ __vector_pair vp = *vpp;
+ vec_t vec = *src;
+ __builtin_mma_dmxvi8gerx4pp (&dmr, vp, vec);
+ dst[1] = dmr;
+}
+
+/* { dg-final { scan-assembler-times {\mdmxvi8gerx4pp\M} 2 } } */
+
+void
+foo_2 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src)
+{
+ __vector_pair vp = *vpp;
+ vec_t vec = *src;
+ __builtin_mma_pmdmxvi8gerx4 (dst, vp, vec, 255, 15, 2);
+}
+
+/* { dg-final { scan-assembler-times {\mpmdmxvi8gerx4\M} 1 } } */
+
+void
+foo_3 (__dmr1024 *dst, __vector_pair *vpp, vec_t *src)
+{
+ __vector_pair vp = *vpp;
+ vec_t vec = *src;
+ __builtin_mma_pmdmxvi8gerx4pp (dst, vp, vec, 255, 15, 2);
+}
+
+/* { dg-final { scan-assembler-times {\mpmdmxvi8gerx4pp\M} 1 } } */
+
+
+void
+foo_5 (__dmr1024 *dst, __dmr1024 *src)
+{
+ __builtin_dmxor (dst, src);
+}
+
+/* { dg-final { scan-assembler-times {\mdmxor\M} 1 } } */
--
2.52.0
