https://github.com/arsenm created
https://github.com/llvm/llvm-project/pull/153021
The first case currently works as expected, but defends against
a regression I nearly introduced in a later patch. The loop
case does not work as expected yet.
>From 814f781a353995c6bdd176c3c6d5effa1af0d611 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <matthew.arsena...@amd.com>
Date: Mon, 11 Aug 2025 21:02:55 +0900
Subject: [PATCH] AMDGPU: Add baseline test for mfma rewrite with phi
The first case currently works as expected, but defends against
a regression I nearly introduced in a later patch. The loop
case does not work as expected yet.
---
.../AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll | 181 ++++++++++++++++++
1 file changed, 181 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll
b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll
new file mode 100644
index 0000000000000..cefcd7e0d2651
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
UTC_ARGS: --version 5
+; RUN: llc -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck %s
+
+target triple = "amdgcn-amd-amdhsa"
+
+define amdgpu_kernel void @test_rewrite_mfma_copy_to_agpr_phi(ptr addrspace(1)
%arg0, ptr addrspace(1) %arg1, i1 %cond) #0 {
+; CHECK-LABEL: test_rewrite_mfma_copy_to_agpr_phi:
+; CHECK: ; %bb.0: ; %bb
+; CHECK-NEXT: s_load_dword s6, s[4:5], 0x10
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_bitcmp0_b32 s6, 0
+; CHECK-NEXT: s_cbranch_scc0 .LBB0_2
+; CHECK-NEXT: ; %bb.1: ; %else
+; CHECK-NEXT: global_load_dwordx4 a[28:31], v32, s[2:3] offset:112
+; CHECK-NEXT: global_load_dwordx4 a[24:27], v32, s[2:3] offset:96
+; CHECK-NEXT: global_load_dwordx4 a[20:23], v32, s[2:3] offset:80
+; CHECK-NEXT: global_load_dwordx4 a[16:19], v32, s[2:3] offset:64
+; CHECK-NEXT: global_load_dwordx4 a[12:15], v32, s[2:3] offset:48
+; CHECK-NEXT: global_load_dwordx4 a[8:11], v32, s[2:3] offset:32
+; CHECK-NEXT: global_load_dwordx4 a[4:7], v32, s[2:3] offset:16
+; CHECK-NEXT: global_load_dwordx4 a[0:3], v32, s[2:3]
+; CHECK-NEXT: v_mov_b32_e32 v33, 2.0
+; CHECK-NEXT: v_mov_b32_e32 v34, 4.0
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 a[0:31], v33, v34, a[0:31]
+; CHECK-NEXT: ; kill: def
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
killed
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
killed $exec
+; CHECK-NEXT: s_cbranch_execz .LBB0_3
+; CHECK-NEXT: s_branch .LBB0_4
+; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT: ; implicit-def:
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+; CHECK-NEXT: .LBB0_3: ; %if
+; CHECK-NEXT: s_nop 7
+; CHECK-NEXT: s_nop 7
+; CHECK-NEXT: global_load_dwordx4 a[28:31], v32, s[0:1] offset:112
+; CHECK-NEXT: global_load_dwordx4 a[24:27], v32, s[0:1] offset:96
+; CHECK-NEXT: global_load_dwordx4 a[20:23], v32, s[0:1] offset:80
+; CHECK-NEXT: global_load_dwordx4 a[16:19], v32, s[0:1] offset:64
+; CHECK-NEXT: global_load_dwordx4 a[12:15], v32, s[0:1] offset:48
+; CHECK-NEXT: global_load_dwordx4 a[8:11], v32, s[0:1] offset:32
+; CHECK-NEXT: global_load_dwordx4 a[4:7], v32, s[0:1] offset:16
+; CHECK-NEXT: global_load_dwordx4 a[0:3], v32, s[0:1]
+; CHECK-NEXT: v_mov_b32_e32 v32, 2.0
+; CHECK-NEXT: v_mov_b32_e32 v33, 4.0
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 a[0:31], v32, v33, a[0:31]
+; CHECK-NEXT: ; kill: def
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
killed
$agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
killed $exec
+; CHECK-NEXT: .LBB0_4: ; %endif
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use a[0:31]
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_endpgm
+bb:
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ br i1 %cond, label %if, label %else
+
+if:
+ %gep.0 = getelementptr <32 x float>, ptr addrspace(1) %arg0, i32 %id
+ %in.0 = load <32 x float>, ptr addrspace(1) %gep.0, align 128
+ %mai.0 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float
4.0, <32 x float> %in.0, i32 0, i32 0, i32 0)
+ br label %endif
+
+else:
+ %gep.1 = getelementptr <32 x float>, ptr addrspace(1) %arg1, i32 %id
+ %in.1 = load <32 x float>, ptr addrspace(1) %gep.1, align 128
+ %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float
4.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
+ br label %endif
+
+endif:
+ %phi = phi <32 x float> [ %mai.0, %if ], [ %mai.1, %else ]
+ call void asm sideeffect "; use $0", "a"(<32 x float> %phi)
+ ret void
+}
+
+define amdgpu_kernel void @test_rewrite_mfma_copy_to_agpr_phi_loop(ptr
addrspace(1) %arg0, ptr addrspace(1) %arg1, i32 %n) #0 {
+; CHECK-LABEL: test_rewrite_mfma_copy_to_agpr_phi_loop:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
+; CHECK-NEXT: s_load_dword s0, s[4:5], 0x10
+; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT: v_lshlrev_b32_e32 v32, 7, v0
+; CHECK-NEXT: s_mov_b32 s1, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: global_load_dwordx4 v[28:31], v32, s[2:3] offset:112
+; CHECK-NEXT: global_load_dwordx4 v[24:27], v32, s[2:3] offset:96
+; CHECK-NEXT: global_load_dwordx4 v[20:23], v32, s[2:3] offset:80
+; CHECK-NEXT: global_load_dwordx4 v[16:19], v32, s[2:3] offset:64
+; CHECK-NEXT: global_load_dwordx4 v[12:15], v32, s[2:3] offset:48
+; CHECK-NEXT: global_load_dwordx4 v[8:11], v32, s[2:3] offset:32
+; CHECK-NEXT: global_load_dwordx4 v[4:7], v32, s[2:3] offset:16
+; CHECK-NEXT: global_load_dwordx4 v[0:3], v32, s[2:3]
+; CHECK-NEXT: v_mov_b32_e32 v64, 4.0
+; CHECK-NEXT: v_mov_b32_e32 v65, 2.0
+; CHECK-NEXT: .LBB1_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_nop 7
+; CHECK-NEXT: s_nop 7
+; CHECK-NEXT: v_mov_b64_e32 v[62:63], v[30:31]
+; CHECK-NEXT: v_mov_b64_e32 v[60:61], v[28:29]
+; CHECK-NEXT: v_mov_b64_e32 v[58:59], v[26:27]
+; CHECK-NEXT: v_mov_b64_e32 v[56:57], v[24:25]
+; CHECK-NEXT: v_mov_b64_e32 v[54:55], v[22:23]
+; CHECK-NEXT: v_mov_b64_e32 v[52:53], v[20:21]
+; CHECK-NEXT: v_mov_b64_e32 v[50:51], v[18:19]
+; CHECK-NEXT: v_mov_b64_e32 v[48:49], v[16:17]
+; CHECK-NEXT: v_mov_b64_e32 v[46:47], v[14:15]
+; CHECK-NEXT: v_mov_b64_e32 v[44:45], v[12:13]
+; CHECK-NEXT: v_mov_b64_e32 v[42:43], v[10:11]
+; CHECK-NEXT: v_mov_b64_e32 v[40:41], v[8:9]
+; CHECK-NEXT: v_mov_b64_e32 v[38:39], v[6:7]
+; CHECK-NEXT: v_mov_b64_e32 v[36:37], v[4:5]
+; CHECK-NEXT: v_mov_b64_e32 v[34:35], v[2:3]
+; CHECK-NEXT: v_mov_b64_e32 v[32:33], v[0:1]
+; CHECK-NEXT: s_add_i32 s1, s1, 1
+; CHECK-NEXT: s_cmp_lt_u32 s1, s0
+; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v65, v64, v[32:63]
+; CHECK-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v65, v64, v[0:31]
+; CHECK-NEXT: s_cbranch_scc1 .LBB1_1
+; CHECK-NEXT: ; %bb.2: ; %endif
+; CHECK-NEXT: v_accvgpr_write_b32 a0, v32
+; CHECK-NEXT: v_accvgpr_write_b32 a1, v33
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v34
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v35
+; CHECK-NEXT: v_accvgpr_write_b32 a4, v36
+; CHECK-NEXT: v_accvgpr_write_b32 a5, v37
+; CHECK-NEXT: v_accvgpr_write_b32 a6, v38
+; CHECK-NEXT: v_accvgpr_write_b32 a7, v39
+; CHECK-NEXT: v_accvgpr_write_b32 a8, v40
+; CHECK-NEXT: v_accvgpr_write_b32 a9, v41
+; CHECK-NEXT: v_accvgpr_write_b32 a10, v42
+; CHECK-NEXT: v_accvgpr_write_b32 a11, v43
+; CHECK-NEXT: v_accvgpr_write_b32 a12, v44
+; CHECK-NEXT: v_accvgpr_write_b32 a13, v45
+; CHECK-NEXT: v_accvgpr_write_b32 a14, v46
+; CHECK-NEXT: v_accvgpr_write_b32 a15, v47
+; CHECK-NEXT: v_accvgpr_write_b32 a16, v48
+; CHECK-NEXT: v_accvgpr_write_b32 a17, v49
+; CHECK-NEXT: v_accvgpr_write_b32 a18, v50
+; CHECK-NEXT: v_accvgpr_write_b32 a19, v51
+; CHECK-NEXT: v_accvgpr_write_b32 a20, v52
+; CHECK-NEXT: v_accvgpr_write_b32 a21, v53
+; CHECK-NEXT: v_accvgpr_write_b32 a22, v54
+; CHECK-NEXT: v_accvgpr_write_b32 a23, v55
+; CHECK-NEXT: v_accvgpr_write_b32 a24, v56
+; CHECK-NEXT: v_accvgpr_write_b32 a25, v57
+; CHECK-NEXT: v_accvgpr_write_b32 a26, v58
+; CHECK-NEXT: v_accvgpr_write_b32 a27, v59
+; CHECK-NEXT: v_accvgpr_write_b32 a28, v60
+; CHECK-NEXT: v_accvgpr_write_b32 a29, v61
+; CHECK-NEXT: v_accvgpr_write_b32 a30, v62
+; CHECK-NEXT: v_accvgpr_write_b32 a31, v63
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use a[0:31]
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_endpgm
+entry:
+ %id = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep.0 = getelementptr <32 x float>, ptr addrspace(1) %arg0, i32 %id
+ %in.0 = load <32 x float>, ptr addrspace(1) %gep.0, align 128
+ br label %loop
+
+loop:
+ %i.phi = phi i32 [ 0, %entry ], [ %i.inc, %loop ]
+ %phi = phi <32 x float> [ %in.0, %entry ], [ %mai.1, %loop ]
+ %mai.0 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float
4.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+ %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float
4.0, <32 x float> %mai.0, i32 0, i32 0, i32 0)
+ %i.inc = add i32 %i.phi, 1
+ %loop.cond = icmp ult i32 %i.inc, %n
+ br i1 %loop.cond, label %loop, label %endif
+
+endif:
+ call void asm sideeffect "; use $0", "a"(<32 x float> %phi)
+ ret void
+}
+
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,256"
"amdgpu-waves-per-eu"="1,1" }
_______________________________________________
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
