================
@@ -56,3 +57,166 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950) {
EXPECT_EQ(Info.Slots[0].Mask, CoExecMask::None) << MCII->getName(Op);
}
}
+
+namespace {
+struct IssueCycles {
+ unsigned MFMAOpc;
+ int64_t CBSZ, BLGP;
+ unsigned VALU, PackedF32;
+};
+
+class MFMAIssueCycleTest {
+public:
+ explicit MFMAIssueCycleTest(std::unique_ptr<GCNTargetMachine> TheTM)
+ : TM(std::move(TheTM)),
+ ST(TM->getTargetTriple(), std::string(TM->getTargetCPU()),
+ std::string(TM->getTargetFeatureString()), *TM),
+ Mod("Module", Ctx), MMI(TM.get()) {
+ Mod.setDataLayout(TM->getTargetTriple().computeDataLayout());
+ auto *F = Function::Create(FunctionType::get(Type::getVoidTy(Ctx), false),
+ GlobalValue::ExternalLinkage, "Test", &Mod);
+ MF = std::make_unique<MachineFunction>(*F, *TM, ST, MMI.getContext(), 0);
+ }
+
+ // Builds an MFMA with placeholder operands; CBSZ/BLGP select F8F6F4 formats.
+ const MachineInstr &buildMFMA(unsigned Opc, int64_t CBSZ, int64_t BLGP) {
+ const SIInstrInfo &TII = *ST.getInstrInfo();
+ const MCInstrDesc &Desc = TII.get(Opc);
+ MachineInstr *MI = MF->CreateMachineInstr(Desc, DebugLoc());
+ for (auto [Idx, Info] : enumerate(Desc.operands())) {
+ if (Info.RegClass == -1)
+ MI->addOperand(*MF, MachineOperand::CreateImm(0));
+ else
+ MI->addOperand(*MF, MachineOperand::CreateReg(Register(), Idx < 1));
+ }
+ if (MachineOperand *Op = TII.getNamedOperand(*MI, AMDGPU::OpName::cbsz))
+ Op->setImm(CBSZ);
+ if (MachineOperand *Op = TII.getNamedOperand(*MI, AMDGPU::OpName::blgp))
+ Op->setImm(BLGP);
+ return *MI;
+ }
+
+ void check(ArrayRef<IssueCycles> Cases) {
+ const SIInstrInfo &TII = *ST.getInstrInfo();
+ for (const IssueCycles &C : Cases) {
+ const MachineInstr &MFMA = buildMFMA(C.MFMAOpc, C.CBSZ, C.BLGP);
+ StringRef Name = TII.getName(C.MFMAOpc);
+ EXPECT_EQ(getVALUIssueCycle(MFMA, TII), C.VALU) << Name;
+ EXPECT_EQ(getPackedVALUIssueCycle(MFMA, V_PK_ADD_F16, TII), C.VALU)
+ << Name;
+ EXPECT_EQ(getPackedVALUIssueCycle(MFMA, V_PK_FMA_F32, TII), C.PackedF32)
+ << Name;
+ EXPECT_EQ(getPackedVALUIssueCycle(MFMA, V_DOT2_F32_F16, TII),
C.PackedF32)
+ << Name;
+ EXPECT_EQ(getPackedVALUIssueCycle(MFMA, /*IsFMALike=*/true,
+ /*IsDOTLike=*/false, TII),
+ C.PackedF32)
+ << Name;
+ }
+ }
+
+ void checkPacked(unsigned MFMAOpc, ArrayRef<unsigned> PackedOpcs,
+ unsigned Expected) {
+ const SIInstrInfo &TII = *ST.getInstrInfo();
+ const MachineInstr &MFMA = buildMFMA(MFMAOpc, 0, 0);
+ for (unsigned PK : PackedOpcs)
+ EXPECT_EQ(getPackedVALUIssueCycle(MFMA, PK, TII), Expected)
+ << TII.getName(PK);
+ }
+
+private:
+ std::unique_ptr<GCNTargetMachine> TM;
+ GCNSubtarget ST;
+ LLVMContext Ctx;
+ Module Mod;
+ MachineModuleInfo MMI;
+ std::unique_ptr<MachineFunction> MF;
+};
+} // namespace
+
+// Expected cycles were measured with ATT on MI308X with the peak TOPS limiter
+// enabled and on MI350X; one cycle is 4 clocks. Scaled MFMAs were measured
+// issuing right behind another MFMA.
+TEST_F(AMDGPUMFMACoExecRules, GFX942IssueCycles) {
+ auto TM =
+ createAMDGPUTargetMachine(Triple("amdgcn-amd-amdhsa"), "gfx942", "");
+ if (!TM)
+ GTEST_SKIP();
+ MFMAIssueCycleTest Test(std::move(TM));
+ Test.checkPacked(V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64,
+ {V_PK_MUL_F32, V_PK_ADD_F32, V_PK_MOV_B32, V_DOT2_I32_I16},
+ 6);
+ Test.checkPacked(V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64,
+ {V_PK_FMA_F16, V_PK_MUL_F16, V_PK_ADD_U16, V_PK_MUL_LO_U16},
+ 1);
+ Test.checkPacked(V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32}, 6);
+ Test.checkPacked(V_MFMA_F32_32X32X8BF16_1K_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32}, 10);
+ Test.checkPacked(V_MFMA_F32_16X16X4F32_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32}, 10);
+ Test.check({
+ {V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64, 0, 0, 1, 6},
+ {V_MFMA_F32_16X16X16BF16_1K_e64, 0, 0, 1, 6},
+ {V_MFMA_F32_4X4X4BF16_1K_vgprcd_e64, 0, 0, 1, 4},
+ {V_MFMA_F32_32X32X8BF16_1K_vgprcd_e64, 0, 0, 1, 10},
+ {V_MFMA_F32_16X16X16F16_vgprcd_e64, 0, 0, 1, 6},
+ {V_MFMA_I32_16X16X32I8_vgprcd_e64, 0, 0, 1, 6},
+ {V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64, 0, 0, 1, 6},
+ {V_MFMA_F32_16X16X8XF32_vgprcd_e64, 0, 0, 1, 6},
+ {V_SMFMAC_F32_16X16X32_F16_e64, 0, 0, 2, 6},
+ {V_MFMA_F32_16X16X4F32_vgprcd_e64, 0, 0, 10, 10},
+ {V_MFMA_F64_16X16X4F64_vgprcd_e64, 0, 0, 10, 10},
+ {V_MFMA_F64_4X4X4F64_vgprcd_e64, 0, 0, 6, 6},
+ });
+}
+
+TEST_F(AMDGPUMFMACoExecRules, GFX950IssueCycles) {
+ auto TM =
+ createAMDGPUTargetMachine(Triple("amdgcn-amd-amdhsa"), "gfx950", "");
+ if (!TM)
+ GTEST_SKIP();
+ MFMAIssueCycleTest Test(std::move(TM));
+ Test.checkPacked(
+ V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32, V_DOT2C_F32_BF16_e32}, 6);
+ Test.checkPacked(V_MFMA_F32_32X32X8BF16_1K_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32}, 10);
+ Test.checkPacked(V_MFMA_F32_16X16X4F32_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32}, 10);
+ Test.checkPacked(
+ V_MFMA_F32_16X16X32_BF16_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32, V_DOT2C_F32_BF16_e32}, 7);
+ Test.checkPacked(V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32}, 11);
+ Test.checkPacked(V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64,
+ {V_DOT2C_F32_F16_e32, V_DOT4C_I32_I8_e32}, 11);
+ Test.check({
+ {V_MFMA_F32_16X16X16BF16_1K_vgprcd_e64, 0, 0, 1, 6},
+ {V_MFMA_F32_4X4X4BF16_1K_vgprcd_e64, 0, 0, 1, 4},
+ {V_MFMA_F32_32X32X8BF16_1K_vgprcd_e64, 0, 0, 1, 10},
+ {V_SMFMAC_F32_16X16X32_F16_e64, 0, 0, 2, 6},
+ {V_MFMA_F32_16X16X32_BF16_vgprcd_e64, 0, 0, 2, 7},
+ {V_MFMA_F32_16X16X32_BF16_e64, 0, 0, 2, 7},
+ {V_MFMA_F32_16X16X32_F16_vgprcd_e64, 0, 0, 2, 7},
+ {V_MFMA_I32_16X16X64_I8_vgprcd_e64, 0, 0, 2, 7},
+ {V_MFMA_F32_32X32X16_BF16_vgprcd_e64, 0, 0, 2, 11},
+ {V_MFMA_F32_32X32X16_F16_vgprcd_e64, 0, 0, 2, 11},
+ {V_MFMA_I32_32X32X32_I8_vgprcd_e64, 0, 0, 2, 11},
+ {V_MFMA_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64, 4, 4, 2, 7},
+ {V_MFMA_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64, 2, 2, 2, 7},
+ {V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64, 0, 0, 3, 11},
+ {V_MFMA_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64, 0, 4, 3, 11},
+ {V_MFMA_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64, 4, 4, 2, 11},
+ {V_MFMA_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64, 0, 0, 3, 19},
+ {V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64, 0, 0, 3, 11},
+ {V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64, 4, 4, 2, 7},
+ {V_SMFMAC_F32_16X16X64_BF16_e64, 0, 0, 3, 7},
----------------
lijinpei-amd wrote:
Agreed, this PR leaves `getMFMACoExecInfo` alone. @jrbyrnes, could you take a
look at whether the SMFMAC VALU slot (3) and the 32x32 SMFMAC occupancy (8) in
the table should follow the measurements above? Happy to send the follow-up if
so.
https://github.com/llvm/llvm-project/pull/233721
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits