https://github.com/rampitec updated https://github.com/llvm/llvm-project/pull/203143
>From d67eca21098240fbf656543bd3d30e120c674c14 Mon Sep 17 00:00:00 2001 From: Stanislav Mekhanoshin <[email protected]> Date: Wed, 10 Jun 2026 17:13:30 -0700 Subject: [PATCH] [AMDGPU] Intrinsic and codegen for wmma_f64_16x16x4_f64 --- llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 6 +- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 23 +++ .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 1 + llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 4 + .../UniformityAnalysis/AMDGPU/intrinsics.ll | 7 + .../AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll | 61 ++++++++ .../llvm.amdgcn.wmma.imm.gfx1251.w32.ll | 144 +++++++++++++++++ .../llvm.amdgcn.wmma.imod.gfx1251.w32.ll | 145 ++++++++++++++++++ 8 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index f4f0fa0d5b1bb..f1659f0cd803a 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -4054,8 +4054,11 @@ class AMDGPUWmmaIntrinsicModsAll<LLVMType AB, LLVMType CD> : LLVMMatchType<1>, // %B llvm_i16_ty, // %C_mod: 0 -- none, 1 -- neg, 2 -- abs, 3 -- neg(abs) LLVMMatchType<0>, // %C + llvm_i1_ty, // matrix_a_reuse + llvm_i1_ty, // matrix_b_reuse ], - [IntrNoMem, IntrConvergent, ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree] + [IntrNoMem, IntrConvergent, ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<6>>, + ImmArg<ArgIndex<7>>, ImmArg<ArgIndex<4>>, IntrWillReturn, IntrNoCallback, IntrNoFree] >; // D and C are of different types. @@ -4135,6 +4138,7 @@ class AMDGPUWmmaScaleF4IntrinsicModsC<LLVMType scale_ty> : >; defset list<Intrinsic> AMDGPUWMMAIntrinsicsGFX1250 = { +def int_amdgcn_wmma_f64_16x16x4_f64 : AMDGPUWmmaIntrinsicModsAll<llvm_anyfloat_ty, llvm_anyfloat_ty>; def int_amdgcn_wmma_f32_16x16x4_f32 : AMDGPUWmmaIntrinsicModsC<llvm_anyfloat_ty, llvm_anyfloat_ty>; def int_amdgcn_wmma_f32_16x16x32_bf16 : AMDGPUWmmaIntrinsicModsC<llvm_anyfloat_ty, llvm_anyfloat_ty>; def int_amdgcn_wmma_f32_16x16x32_f16 : AMDGPUWmmaIntrinsicModsC<llvm_anyfloat_ty, llvm_anyfloat_ty>; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 981d880bcf7b0..b545ec82ac8b7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -4086,6 +4086,29 @@ bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(SDValue In, SDValue &Src) const { } } + // Currently f64 immediate vectors are represented as vectors of v2i32, with + // different lo and hi 32-bit values even though double values are splated. + // So we have to manually compare to determine whether it is splated. + if (CurDAG->isConstantIntBuildVectorOrConstantInt(SplatSrc32)) { + int64_t Imm64 = 0; + for (unsigned i = 0; i < SplatSrc32->getNumOperands(); i += 2) { + auto Lo32 = cast<ConstantSDNode>(SplatSrc32->getOperand(i)); + auto Hi32 = cast<ConstantSDNode>(SplatSrc32->getOperand(i + 1)); + int64_t LoImm = Lo32->getAPIntValue().getSExtValue(); + int64_t HiImm = Hi32->getAPIntValue().getSExtValue(); + int64_t Imm64I = (HiImm << 32) + LoImm; + if (i == 0) { + if (!isInlineImmediate(APInt(64, Imm64I))) + return false; + Imm64 = Imm64I; + } else if (Imm64I != Imm64) + return false; + } // end for + + Src = CurDAG->getTargetConstant(Imm64, SDLoc(In), MVT::i64); + return true; + } + return false; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index cc1f2d0664484..db8fc149df1f6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -4826,6 +4826,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8: case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8: case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: + case Intrinsic::amdgcn_wmma_f64_16x16x4_f64: case Intrinsic::amdgcn_wmma_f32_16x16x4_f32: case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16: case Intrinsic::amdgcn_wmma_f32_16x16x32_f16: diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td index add70e1d62222..b5798ec0c7078 100644 --- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -2256,6 +2256,10 @@ let SubtargetPredicate = isGFX125xOnly in { def : SWMMACPat<V_SWMMAC_F32_16X16X64_F16_w32_twoaddr, int_amdgcn_swmmac_f32_16x16x64_f16, F32_F16X64_SWMMAC_w32>; def : SWMMACPat<V_SWMMAC_F16_16X16X64_F16_w32_twoaddr, int_amdgcn_swmmac_f16_16x16x64_f16, F16_F16X64_SWMMAC_w32>; } // End SubtargetPredicate = isGFX125xOnly + +let SubtargetPredicate = HasGFX1251GEMMInsts in { + defm : WMMAPat<"V_WMMA_F64_16X16X4_F64_w32", int_amdgcn_wmma_f64_16x16x4_f64, F64_F64X4_WMMA_w32>; +} // End SubtargetPredicate = HasGFX1251GEMMInsts } // End WaveSizePredicate = isWave32 //===----------------------------------------------------------------------===// diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll index 9c57f1f2e5367..73d09a9dad37f 100644 --- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll +++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll @@ -204,6 +204,13 @@ bb: ret void } +; CHECK: DIVERGENT: %tmp0 = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 false, <2 x double> %A, i1 false, <2 x double> %B, i16 0, <8 x double> %C, i1 false, i1 false) +define amdgpu_kernel void @wmma_f64_16x16x4_f64(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { + %tmp0 = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 0, <8 x double> %C, i1 false, i1 false) + store <8 x double> %tmp0, ptr addrspace(1) %out + ret void +} + ; CHECK: DIVERGENT: %tmp0 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(<2 x float> %A, <2 x float> %B, i16 0, <8 x float> %C, i1 false, i1 false) define amdgpu_kernel void @wmma_f32_16x16x4_f32(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) { %tmp0 = call <8 x float> @llvm.amdgcn.wmma.f32.16x16x4.f32.v8f32.v2f32(<2 x float> %A, <2 x float> %B, i16 0, <8 x float> %C, i1 false, i1 false) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll new file mode 100644 index 0000000000000..9b4d2df8148fb --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll @@ -0,0 +1,61 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1251 < %s | FileCheck %s --check-prefix=GFX1251 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1251 -global-isel < %s | FileCheck %s --check-prefix=GISEL + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_matrix_a_reuse(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_matrix_a_reuse: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_a_reuse +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GFX1251-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GFX1251-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GFX1251-NEXT: global_store_b128 v[24:25], v[8:11], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_matrix_a_reuse: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_a_reuse +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[24:25], v[8:11], off +; GISEL-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 0, <8 x double> %C, i1 true, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_matrix_b_reuse(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_matrix_b_reuse: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_b_reuse +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GFX1251-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GFX1251-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GFX1251-NEXT: global_store_b128 v[24:25], v[8:11], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_matrix_b_reuse: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_b_reuse +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[24:25], v[8:11], off +; GISEL-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 0, <8 x double> %C, i1 false, i1 true) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +declare <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1, <2 x double>, i1, <2 x double>, i16, <8 x double>, i1, i1) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll new file mode 100644 index 0000000000000..f654d63b70a63 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll @@ -0,0 +1,144 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1251 < %s | FileCheck %s --check-prefix=GFX1251 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1251 -global-isel < %s | FileCheck %s --check-prefix=GISEL + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64(<2 x double> %A, <2 x double> %B, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], 1.0 +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[8:9], v[22:25], off offset:48 +; GFX1251-NEXT: global_store_b128 v[8:9], v[18:21], off offset:32 +; GFX1251-NEXT: global_store_b128 v[8:9], v[14:17], off offset:16 +; GFX1251-NEXT: global_store_b128 v[8:9], v[10:13], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], 1.0 +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[8:9], v[10:13], off +; GISEL-NEXT: global_store_b128 v[8:9], v[14:17], off offset:16 +; GISEL-NEXT: global_store_b128 v[8:9], v[18:21], off offset:32 +; GISEL-NEXT: global_store_b128 v[8:9], v[22:25], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 0, <8 x double> <double 1.0, double 1.0, double 1.0, double 1.0, double 1.0, double 1.0, double 1.0, double 1.0>, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_non_splat(<2 x double> %A, <2 x double> %B, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_non_splat: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0x3ff00000 +; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1251-NEXT: v_dual_mov_b32 v15, 2.0 :: v_dual_mov_b32 v12, v10 +; GFX1251-NEXT: v_dual_mov_b32 v13, v11 :: v_dual_mov_b32 v14, v10 +; GFX1251-NEXT: v_dual_mov_b32 v16, v10 :: v_dual_mov_b32 v17, v11 +; GFX1251-NEXT: v_dual_mov_b32 v18, v10 :: v_dual_mov_b32 v19, v11 +; GFX1251-NEXT: v_dual_mov_b32 v20, v10 :: v_dual_mov_b32 v21, v11 +; GFX1251-NEXT: v_dual_mov_b32 v22, v10 :: v_dual_mov_b32 v23, v11 +; GFX1251-NEXT: v_dual_mov_b32 v24, v10 :: v_dual_mov_b32 v25, v11 +; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], v[10:25] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[8:9], v[22:25], off offset:48 +; GFX1251-NEXT: global_store_b128 v[8:9], v[18:21], off offset:32 +; GFX1251-NEXT: global_store_b128 v[8:9], v[14:17], off offset:16 +; GFX1251-NEXT: global_store_b128 v[8:9], v[10:13], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_non_splat: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: s_mov_b64 s[0:1], 1.0 +; GISEL-NEXT: s_mov_b64 s[4:5], 2.0 +; GISEL-NEXT: s_mov_b64 s[14:15], s[0:1] +; GISEL-NEXT: s_mov_b64 s[2:3], s[0:1] +; GISEL-NEXT: s_mov_b64 s[6:7], s[0:1] +; GISEL-NEXT: s_mov_b64 s[8:9], s[0:1] +; GISEL-NEXT: s_mov_b64 s[10:11], s[0:1] +; GISEL-NEXT: s_mov_b64 s[12:13], s[0:1] +; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[14:15] +; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[12:13] +; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[10:11] +; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[6:7] +; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[4:5] +; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[2:3] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1] +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], v[10:25] +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[8:9], v[10:13], off +; GISEL-NEXT: global_store_b128 v[8:9], v[14:17], off offset:16 +; GISEL-NEXT: global_store_b128 v[8:9], v[18:21], off offset:32 +; GISEL-NEXT: global_store_b128 v[8:9], v[22:25], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 0, <8 x double> <double 1.0, double 1.0, double 2.0, double 1.0, double 1.0, double 1.0, double 1.0, double 1.0>, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_non_inlineable(<2 x double> %A, <2 x double> %B, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_non_inlineable: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0x40080000 +; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1251-NEXT: v_dual_mov_b32 v12, v10 :: v_dual_mov_b32 v13, v11 +; GFX1251-NEXT: v_dual_mov_b32 v14, v10 :: v_dual_mov_b32 v15, v11 +; GFX1251-NEXT: v_dual_mov_b32 v16, v10 :: v_dual_mov_b32 v17, v11 +; GFX1251-NEXT: v_dual_mov_b32 v18, v10 :: v_dual_mov_b32 v19, v11 +; GFX1251-NEXT: v_dual_mov_b32 v20, v10 :: v_dual_mov_b32 v21, v11 +; GFX1251-NEXT: v_dual_mov_b32 v22, v10 :: v_dual_mov_b32 v23, v11 +; GFX1251-NEXT: v_dual_mov_b32 v24, v10 :: v_dual_mov_b32 v25, v11 +; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], v[10:25] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[8:9], v[22:25], off offset:48 +; GFX1251-NEXT: global_store_b128 v[8:9], v[18:21], off offset:32 +; GFX1251-NEXT: global_store_b128 v[8:9], v[14:17], off offset:16 +; GFX1251-NEXT: global_store_b128 v[8:9], v[10:13], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_non_inlineable: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: s_mov_b64 s[0:1], 0x4008000000000000 +; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GISEL-NEXT: s_mov_b64 s[14:15], s[0:1] +; GISEL-NEXT: s_mov_b64 s[2:3], s[0:1] +; GISEL-NEXT: s_mov_b64 s[4:5], s[0:1] +; GISEL-NEXT: s_mov_b64 s[6:7], s[0:1] +; GISEL-NEXT: s_mov_b64 s[8:9], s[0:1] +; GISEL-NEXT: s_mov_b64 s[10:11], s[0:1] +; GISEL-NEXT: s_mov_b64 s[12:13], s[0:1] +; GISEL-NEXT: v_mov_b64_e32 v[24:25], s[14:15] +; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[12:13] +; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[10:11] +; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[8:9] +; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[6:7] +; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[4:5] +; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[2:3] +; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1] +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], v[10:25] +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[8:9], v[10:13], off +; GISEL-NEXT: global_store_b128 v[8:9], v[14:17], off offset:16 +; GISEL-NEXT: global_store_b128 v[8:9], v[18:21], off offset:32 +; GISEL-NEXT: global_store_b128 v[8:9], v[22:25], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 0, <8 x double> <double 3.0, double 3.0, double 3.0, double 3.0, double 3.0, double 3.0, double 3.0, double 3.0>, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +declare <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1, <2 x double>, i1, <2 x double>, i16, <8 x double>, i1, i1) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll new file mode 100644 index 0000000000000..fcc3c98e78534 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll @@ -0,0 +1,145 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1251 < %s | FileCheck %s --check-prefix=GFX1251 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1251 -global-isel < %s | FileCheck %s --check-prefix=GISEL + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negA(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_negA: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[1,0,0] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GFX1251-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GFX1251-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GFX1251-NEXT: global_store_b128 v[24:25], v[8:11], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_negA: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[1,0,0] +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[24:25], v[8:11], off +; GISEL-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 1, <2 x double> %A, i1 0, <2 x double> %B, i16 0, <8 x double> %C, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negB(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_negB: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,1,0] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GFX1251-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GFX1251-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GFX1251-NEXT: global_store_b128 v[24:25], v[8:11], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_negB: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,1,0] +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[24:25], v[8:11], off +; GISEL-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 1, <2 x double> %B, i16 0, <8 x double> %C, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negC(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_negC: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GFX1251-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GFX1251-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GFX1251-NEXT: global_store_b128 v[24:25], v[8:11], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_negC: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1] +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[24:25], v[8:11], off +; GISEL-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 1, <8 x double> %C, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_neg_absC(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_neg_absC: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1] neg_hi:[0,0,1] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GFX1251-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GFX1251-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GFX1251-NEXT: global_store_b128 v[24:25], v[8:11], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_neg_absC: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1] neg_hi:[0,0,1] +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[24:25], v[8:11], off +; GISEL-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 3, <8 x double> %C, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +define amdgpu_ps void @test_wmma_f64_16x16x4_f64_ignoreC(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) { +; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_ignoreC: +; GFX1251: ; %bb.0: ; %bb +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GFX1251-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GFX1251-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GFX1251-NEXT: global_store_b128 v[24:25], v[8:11], off +; GFX1251-NEXT: s_endpgm +; +; GISEL-LABEL: test_wmma_f64_16x16x4_f64_ignoreC: +; GISEL: ; %bb.0: ; %bb +; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] +; GISEL-NEXT: s_clause 0x3 +; GISEL-NEXT: global_store_b128 v[24:25], v[8:11], off +; GISEL-NEXT: global_store_b128 v[24:25], v[12:15], off offset:16 +; GISEL-NEXT: global_store_b128 v[24:25], v[16:19], off offset:32 +; GISEL-NEXT: global_store_b128 v[24:25], v[20:23], off offset:48 +; GISEL-NEXT: s_endpgm +bb: + %res = call <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1 0, <2 x double> %A, i1 0, <2 x double> %B, i16 4, <8 x double> %C, i1 false, i1 false) + store <8 x double> %res, ptr addrspace(1) %out + ret void +} + +declare <8 x double> @llvm.amdgcn.wmma.f64.16x16x4.f64.v8f64.v2f64(i1, <2 x double>, i1, <2 x double>, i16, <8 x double>, i1, i1) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
