https://github.com/lijinpei-amd created 
https://github.com/llvm/llvm-project/pull/239095

Depends on #239094.

getMFMACoExecInfo allowed DS, VMEM, SALU or VALU in the wrong cycles after
these gfx950 MFMAs:
- SMFMAC 16x16 holds VALU until cycle 3, not 2. Split them from the XDL
  16x16 entry, which allows VALU from cycle 2.
- SMFMAC 32x32 allows DS, VMEM and VALU from cycle 3, not 4.
- DGEMM 16x16x4 allows DS, VMEM and SALU from cycle 1, not 0.

MI350X issues VALU at cycle 3 after both SMFMAC sizes.

The first commit adds DS, VMEM and SALU expectations to GFX950IssueStages
with the current values; the second updates them with the fix.

Assisted-by: Claude Opus 5.5

>From 61f1ccf465b72e627b824f8b4104b5c7463b7d7a Mon Sep 17 00:00:00 2001
From: Li Jinpei <[email protected]>
Date: Thu, 5 Nov 2026 12:14:52 +0800
Subject: [PATCH 1/2] [AMDGPU] Precommit MFMA co-exec DS, VMEM and SALU stage
 tests (NFC)

Check the first cycle DS, VMEM and SALU can issue after each MFMA in
GFX950IssueStages. The expectations match the current table, which holds DS
and VMEM one cycle too long after SMFMAC 32x32 and allows DS, VMEM and SALU
one cycle too early after DGEMM 16x16x4.

Assisted-by: Claude Opus 5.5
---
 .../Target/AMDGPU/MFMACoExecRules.cpp         | 26 ++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp 
b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
index 48a9a359d392a..c7801421a704f 100644
--- a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
+++ b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
@@ -121,7 +121,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
     // Opcode names without the _e64, _vgprcd_e64 or _mac*_e64 suffix.
     std::vector<StringRef> MFMAs;
     struct {
-      unsigned VALU, PackedF32;
+      unsigned VALU, PackedF32, DS, VMEM, SALU;
     } Expected;
   } Groups[] = {
       {{"V_MFMA_F32_16X16X32_BF16", "V_MFMA_F32_16X16X32_F16",
@@ -133,7 +133,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6"},
-       {2, 7}},
+       {2, 7, 2, 2, 1}},
       {{"V_MFMA_F32_16X16X128_F8F6F4_f4_f8",
         "V_MFMA_F32_16X16X128_F8F6F4_f6_f8",
         "V_MFMA_F32_16X16X128_F8F6F4_f8_f4",
@@ -144,7 +144,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8"},
-       {3, 11}},
+       {3, 11, 2, 2, 1}},
       {{"V_MFMA_F32_32X32X16_BF16", "V_MFMA_F32_32X32X16_F16",
         "V_MFMA_I32_32X32X32_I8", "V_MFMA_F32_32X32X64_F8F6F4_f4_f4",
         "V_MFMA_F32_32X32X64_F8F6F4_f4_f6", "V_MFMA_F32_32X32X64_F8F6F4_f6_f4",
@@ -153,7 +153,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6"},
-       {2, 11}},
+       {2, 11, 2, 2, 1}},
       {{"V_MFMA_F32_32X32X64_F8F6F4_f4_f8", "V_MFMA_F32_32X32X64_F8F6F4_f6_f8",
         "V_MFMA_F32_32X32X64_F8F6F4_f8_f4", "V_MFMA_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_F32_32X32X64_F8F6F4_f8_f8",
@@ -162,20 +162,21 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8"},
-       {3, 19}},
+       {3, 19, 2, 2, 1}},
       // The table holds VALU until 2 (16x16) or 4 (32x32); MI350X issues it
-      // at 3.
+      // at 3. It also holds DS and VMEM until 4 (32x32), not 3.
       {{"V_SMFMAC_F32_16X16X64_BF16", "V_SMFMAC_F32_16X16X64_F16",
         "V_SMFMAC_I32_16X16X128_I8", "V_SMFMAC_F32_16X16X128_BF8_BF8",
         "V_SMFMAC_F32_16X16X128_BF8_FP8", "V_SMFMAC_F32_16X16X128_FP8_BF8",
         "V_SMFMAC_F32_16X16X128_FP8_FP8"},
-       {2, 7}},
+       {2, 7, 2, 2, 1}},
       {{"V_SMFMAC_F32_32X32X32_BF16", "V_SMFMAC_F32_32X32X32_F16",
         "V_SMFMAC_I32_32X32X64_I8", "V_SMFMAC_F32_32X32X64_BF8_BF8",
         "V_SMFMAC_F32_32X32X64_BF8_FP8", "V_SMFMAC_F32_32X32X64_FP8_BF8",
         "V_SMFMAC_F32_32X32X64_FP8_FP8"},
-       {4, 11}},
-      {{"V_MFMA_F64_16X16X4F64"}, {18, 18}},
+       {4, 11, 4, 4, 1}},
+      // The table allows DS, VMEM and SALU at 0, not from 1.
+      {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 0, 0, 0}},
   };
   for (const auto &G : Groups) {
     for (StringRef Base : G.MFMAs) {
@@ -195,6 +196,13 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         EXPECT_EQ(Test.getIssueStage(MFMAOpc, V_PK_FMA_F32),
                   G.Expected.PackedF32)
             << Name;
+        CoExecInfo Info = getMFMACoExecInfo(MFMAOpc);
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::DS), G.Expected.DS)
+            << Name;
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::VMEM), G.Expected.VMEM)
+            << Name;
+        EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::SALU), G.Expected.SALU)
+            << Name;
       }
       EXPECT_GT(NumForms, 0u) << Base;
     }

>From c0c4d2233b5822824dff98d20f08786c538e44af Mon Sep 17 00:00:00 2001
From: Li Jinpei <[email protected]>
Date: Thu, 5 Nov 2026 00:19:00 +0800
Subject: [PATCH 2/2] [AMDGPU] Fix MFMA co-exec DS, VMEM, SALU and VALU stages
 for SMFMAC and DGEMM

getMFMACoExecInfo allowed DS, VMEM, SALU or VALU in the wrong cycles after
these gfx950 MFMAs:
- SMFMAC 16x16 holds VALU until cycle 3, not 2. Split them from the XDL
  16x16 entry, which allows VALU from cycle 2.
- SMFMAC 32x32 allows DS, VMEM and VALU from cycle 3, not 4.
- DGEMM 16x16x4 allows DS, VMEM and SALU from cycle 1, not 0.

MI350X issues VALU at cycle 3 after both SMFMAC sizes.

Update the expected stages in GFX950IssueStages.

Assisted-by: Claude Opus 5.5
---
 .../AMDGPU/AMDGPUCoExecSchedStrategy.cpp       | 18 ++++++++++++------
 .../Target/AMDGPU/MFMACoExecRules.cpp          |  9 +++------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp 
b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
index 0e4441eeca2e3..3beab829f99c5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp
@@ -136,9 +136,14 @@ llvm::AMDGPU::CoExecInfo 
llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
   case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
-    // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
-    // intended purposes here all those instructions are the same. This comment
-    // is to simplify reverse mapping to the SPG.
+    Res.TotalWindow = 8;
+    allowCoExec(Res, CoExecMask::SALU, 1);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::WMMA, 4);
+    allowCoExec(Res, CoExecMask::PK_F32, 7);
+    return Res;
+
+  // SMFMAC, 4-cycle occupancy, 8-cycle window.
   case V_SMFMAC_F32_16X16X64_BF16_e64:
   case V_SMFMAC_F32_16X16X64_BF16_gfx940:
   case V_SMFMAC_I32_16X16X128_I8_e64:
@@ -155,7 +160,8 @@ llvm::AMDGPU::CoExecInfo 
llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_SMFMAC_F32_16X16X64_F16_gfx940:
     Res.TotalWindow = 8;
     allowCoExec(Res, CoExecMask::SALU, 1);
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VMEM, 2);
+    allowCoExec(Res, CoExecMask::VALU, 3);
     allowCoExec(Res, CoExecMask::WMMA, 4);
     allowCoExec(Res, CoExecMask::PK_F32, 7);
     return Res;
@@ -371,7 +377,7 @@ llvm::AMDGPU::CoExecInfo 
llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_SMFMAC_F32_32X32X32_F16_gfx940:
     Res.TotalWindow = 12;
     allowCoExec(Res, CoExecMask::SALU, 1);
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 4);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 3);
     allowCoExec(Res, CoExecMask::WMMA, 9);
     allowCoExec(Res, CoExecMask::PK_F32, 11);
     return Res;
@@ -382,7 +388,7 @@ llvm::AMDGPU::CoExecInfo 
llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
   case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
   case V_MFMA_F64_16X16X4F64_vgprcd_e64:
     Res.TotalWindow = 19;
-    allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 0);
+    allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 1);
     allowCoExec(Res, CoExecMask::WMMA | CoExecMask::VALU | CoExecMask::PK_F32,
                 18);
     return Res;
diff --git a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp 
b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
index c7801421a704f..d67d5e0a2ecae 100644
--- a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
+++ b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp
@@ -163,20 +163,17 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) {
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6",
         "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8"},
        {3, 19, 2, 2, 1}},
-      // The table holds VALU until 2 (16x16) or 4 (32x32); MI350X issues it
-      // at 3. It also holds DS and VMEM until 4 (32x32), not 3.
       {{"V_SMFMAC_F32_16X16X64_BF16", "V_SMFMAC_F32_16X16X64_F16",
         "V_SMFMAC_I32_16X16X128_I8", "V_SMFMAC_F32_16X16X128_BF8_BF8",
         "V_SMFMAC_F32_16X16X128_BF8_FP8", "V_SMFMAC_F32_16X16X128_FP8_BF8",
         "V_SMFMAC_F32_16X16X128_FP8_FP8"},
-       {2, 7, 2, 2, 1}},
+       {3, 7, 2, 2, 1}},
       {{"V_SMFMAC_F32_32X32X32_BF16", "V_SMFMAC_F32_32X32X32_F16",
         "V_SMFMAC_I32_32X32X64_I8", "V_SMFMAC_F32_32X32X64_BF8_BF8",
         "V_SMFMAC_F32_32X32X64_BF8_FP8", "V_SMFMAC_F32_32X32X64_FP8_BF8",
         "V_SMFMAC_F32_32X32X64_FP8_FP8"},
-       {4, 11, 4, 4, 1}},
-      // The table allows DS, VMEM and SALU at 0, not from 1.
-      {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 0, 0, 0}},
+       {3, 11, 3, 3, 1}},
+      {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 1, 1, 1}},
   };
   for (const auto &G : Groups) {
     for (StringRef Base : G.MFMAs) {

_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to