llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-llvm-globalisel Author: Matt Arsenault (arsenm) <details> <summary>Changes</summary> Atomic load and store of vector types are now permitted in the IR. Instead of maintaining separate scalar-only atomic pattern lists, cover atomic load/store in the existing per-register-type pattern loops. As a side effect -flat-for-global is respected in more cases. Co-authored-by: Claude Opus 5.5 <noreply@<!-- -->anthropic.com> --- Patch is 130.09 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/229688.diff 14 Files Affected: - (modified) llvm/lib/Target/AMDGPU/AMDGPUInstructions.td (+6-16) - (modified) llvm/lib/Target/AMDGPU/BUFInstructions.td (+4-18) - (modified) llvm/lib/Target/AMDGPU/DSInstructions.td (+10-10) - (modified) llvm/lib/Target/AMDGPU/FLATInstructions.td (+15-29) - (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.td (+10-33) - (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir (+45-15) - (modified) llvm/test/CodeGen/AMDGPU/global_atomics.ll (+28-34) - (modified) llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll (+18-18) - (modified) llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll (+40) - (modified) llvm/test/CodeGen/AMDGPU/load-atomic-global.ll (+649) - (modified) llvm/test/CodeGen/AMDGPU/load-atomic-local.ll (+333) - (modified) llvm/test/CodeGen/AMDGPU/store-atomic-flat.ll (+24) - (modified) llvm/test/CodeGen/AMDGPU/store-atomic-global.ll (+398) - (modified) llvm/test/CodeGen/AMDGPU/store-atomic-local.ll (+250) ``````````diff diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index 434ad35f73499..069ebf72bc987 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -528,19 +528,11 @@ def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextloadi16 node:$ptr)> { let IsLoad = 1; } -def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> { - let IsAtomic = 1; -} - -def atomic_load_nonext_32_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_32 node:$ptr)> { - let IsAtomic = 1; -} - -def atomic_load_nonext_64_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_64 node:$ptr)> { +def atomic_load_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext node:$ptr)> { let IsAtomic = 1; } -def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_128 node:$ptr)> { +def atomic_load_nonext_16_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_16 node:$ptr)> { let IsAtomic = 1; } @@ -600,16 +592,14 @@ def truncstorei16_hi16_#as : StoreHi16<truncstorei16, i16>; } // End let IsStore = 1, AddressSpaces = ... let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in { +def atomic_store_#as : PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsTruncStore = 0; +} def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr), (atomic_store_8 node:$val, node:$ptr)>; def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr), (atomic_store_16 node:$val, node:$ptr)>; -def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr), - (atomic_store_32 node:$val, node:$ptr)>; -def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr), - (atomic_store_64 node:$val, node:$ptr)>; -def atomic_store_128_#as : PatFrag<(ops node:$val, node:$ptr), - (atomic_store_128 node:$val, node:$ptr)>; } // End let IsAtomic = 1, AddressSpaces = ... } // End foreach as diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 6cb0f77e0dbeb..984adf838a55d 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1084,10 +1084,12 @@ defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, atomic_load_sext_16_glo foreach vt = Reg32Types.types in { defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>; +defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, atomic_load_global>; } foreach vt = VReg_64.RegTypes in { defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>; +defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, atomic_load_global>; } foreach vt = VReg_96.RegTypes in { @@ -1133,10 +1135,12 @@ defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>; foreach vt = Reg32Types.types in { defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>; +defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, atomic_store_global>; } foreach vt = VReg_64.RegTypes in { defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>; +defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, atomic_store_global>; } foreach vt = VReg_96.RegTypes in { @@ -2061,19 +2065,6 @@ class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) >; -multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, - ValueType vt, PatFrag atomic_ld> { - def : GCNPat < - (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), - (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) - >; - - def : GCNPat < - (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))), - (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) - >; -} - let SubtargetPredicate = isGFX6GFX7 in { def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; @@ -2081,9 +2072,6 @@ def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constan def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; - -defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_nonext_32_global>; -defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_nonext_64_global>; } // End SubtargetPredicate = isGFX6GFX7 multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt, @@ -2255,8 +2243,6 @@ defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OF defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; -defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; -defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; } // End Predicates = isGFX6GFX7 diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 6e6d9597e36c9..3ff2ef169ee8f 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -1005,6 +1005,11 @@ defm : DSReadPat_t16 <DS_READ_U16, i16, "load_local">; foreach vt = Reg32Types.types in { defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">; +defm : DSReadPat_mc <DS_READ_B32, vt, "atomic_load_local">; +} + +foreach vt = VReg_64.RegTypes in { +defm : DSReadPat_mc <DS_READ_B64, vt, "atomic_load_local">; } defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_aext_8_local">; @@ -1017,11 +1022,6 @@ defm : DSReadPat_t16 <DS_READ_U16, i16, "atomic_load_nonext_16_local">; defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_aext_16_local">; defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">; defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">; -defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_nonext_32_local">; -defm : DSReadPat_mc <DS_READ_B32, v2i16, "atomic_load_nonext_32_local">; -defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_nonext_64_local">; -defm : DSReadPat_mc <DS_READ_B64, v2i32, "atomic_load_nonext_64_local">; -defm : DSReadPat_mc <DS_READ_B64, v4i16, "atomic_load_nonext_64_local">; let OtherPredicates = [D16PreservesUnusedBits] in { // TODO: Atomic loads @@ -1098,17 +1098,17 @@ defm : DSWritePat_t16 <DS_WRITE_B16, i16, "store_local">; foreach vt = Reg32Types.types in { defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">; +defm : DSWritePat_mc <DS_WRITE_B32, vt, "atomic_store_local">; +} + +foreach vt = VReg_64.RegTypes in { +defm : DSWritePat_mc <DS_WRITE_B64, vt, "atomic_store_local">; } defm : DSWritePat_t16 <DS_WRITE_B8, i16, "atomic_store_8_local">; defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">; defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">; defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">; -defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">; -defm : DSWritePat_mc <DS_WRITE_B32, v2i16, "atomic_store_32_local">; -defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">; -defm : DSWritePat_mc <DS_WRITE_B64, v2i32, "atomic_store_64_local">; -defm : DSWritePat_mc <DS_WRITE_B64, v4i16, "atomic_store_64_local">; let OtherPredicates = [HasD16LoadStore] in { def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>; diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index f241f6677d22b..5a431a27df8a9 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -2107,24 +2107,21 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst defm : FlatStorePats_t16 <FLAT_STORE_SHORT, atomic_store_16_flat, i16>; } -defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, i32>; -defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_nonext_32_flat, v2i16>; -defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, i64>; -defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v4i16>; -defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_nonext_64_flat, v2i32>; -defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_nonext_128_flat, v4i32>; - defm : FlatStorePats <FLAT_STORE_BYTE, truncstorei8_flat, i32>; defm : FlatStorePats <FLAT_STORE_SHORT, truncstorei16_flat, i32>; foreach vt = Reg32Types.types in { defm : FlatLoadPats <FLAT_LOAD_DWORD, load_flat, vt>; defm : FlatStorePats <FLAT_STORE_DWORD, store_flat, vt>; +defm : FlatLoadPats <FLAT_LOAD_DWORD, atomic_load_flat, vt>; +defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_flat, vt>; } foreach vt = VReg_64.RegTypes in { defm : FlatStorePats <FLAT_STORE_DWORDX2, store_flat, vt>; defm : FlatLoadPats <FLAT_LOAD_DWORDX2, load_flat, vt>; +defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_flat, vt>; +defm : FlatLoadPats <FLAT_LOAD_DWORDX2, atomic_load_flat, vt>; } defm : FlatStorePats <FLAT_STORE_DWORDX3, store_flat, v3i32>; @@ -2132,14 +2129,10 @@ defm : FlatStorePats <FLAT_STORE_DWORDX3, store_flat, v3i32>; foreach vt = VReg_128.RegTypes in { defm : FlatLoadPats <FLAT_LOAD_DWORDX4, load_flat, vt>; defm : FlatStorePats <FLAT_STORE_DWORDX4, store_flat, vt>; +defm : FlatLoadPats <FLAT_LOAD_DWORDX4, atomic_load_flat, vt>; +defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_flat, vt>; } -defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, i32>; -defm : FlatStorePats <FLAT_STORE_DWORD, atomic_store_32_flat, v2i16>; -defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>; -defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v2i32>; -defm : FlatStorePats <FLAT_STORE_DWORDX2, atomic_store_64_flat, v4i16>; -defm : FlatStorePats <FLAT_STORE_DWORDX4, atomic_store_128_flat, v4i32>; defm : FlatStorePats <FLAT_STORE_BYTE, atomic_store_8_flat, i32>; defm : FlatStorePats <FLAT_STORE_SHORT, atomic_store_16_flat, i32>; @@ -2271,14 +2264,21 @@ defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_BYTE", atomic_store_8_global, i defm : GlobalFLATStorePats_D16_t16<"GLOBAL_STORE_SHORT", atomic_store_16_global, i16>; } // end OtherPredicates = [HasFlatGlobalInsts, D16PreservesUnusedBits], True16Predicate = UseRealTrue16Insts +// There is no distinction for atomic load lowering during selection; +// the memory legalizer will set the cache bits and insert the +// appropriate waits. foreach vt = Reg32Types.types in { defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, load_global, vt>; defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, store_global, vt>; +defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_global, vt>; +defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_global, vt>; } foreach vt = VReg_64.RegTypes in { defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, load_global, vt>; defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, store_global, vt>; +defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_global, vt>; +defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_global, vt>; } defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; @@ -2286,18 +2286,10 @@ defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; foreach vt = VReg_128.RegTypes in { defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, load_global, vt>; defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, store_global, vt>; +defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_global, vt>; +defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_global, vt>; } -// There is no distinction for atomic load lowering during selection; -// the memory legalizer will set the cache bits and insert the -// appropriate waits. -defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, i32>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORD, atomic_load_nonext_32_global, v2i16>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, i64>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v2i32>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX2, atomic_load_nonext_64_global, v4i16>; -defm : GlobalFLATLoadPats <GLOBAL_LOAD_DWORDX4, atomic_load_nonext_128_global, v4i32>; - defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, truncstorei8_global, i32>; defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, truncstorei16_global, i32>; defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX3, store_global, v3i32>; @@ -2340,12 +2332,6 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16> defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>; defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>; -defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>; -defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, v2i16>; -defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>; -defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v4i16>; -defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, v2i32>; -defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX4, atomic_store_128_global, v4i32>; defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>; defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index e7425708d51a0..240d03478dc75 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -465,18 +465,6 @@ def atomic_load_nonext_16_glue : PatFrag<(ops node:$ptr), let MemoryVT = i16; } -def atomic_load_nonext_32_glue : PatFrag<(ops node:$ptr), - (atomic_load_nonext_glue node:$ptr)> { - let IsAtomic = 1; - let MemoryVT = i32; -} - -def atomic_load_nonext_64_glue : PatFrag<(ops node:$ptr), - (atomic_load_nonext_glue node:$ptr)> { - let IsAtomic = 1; - let MemoryVT = i64; -} - def atomic_load_zext_8_glue : PatFrag<(ops node:$ptr), (atomic_load_zext_glue node:$ptr)> { let IsAtomic = 1; @@ -587,12 +575,10 @@ def load_align16_#as#_m0 : PatFrag<(ops node:$ptr), } let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in { +def atomic_load_#as#_m0 : PatFrag<(ops node:$ptr), + (atomic_load_nonext_glue node:$ptr)>; def atomic_load_nonext_16_#as#_m0 : PatFrag<(ops node:$ptr), (atomic_load_nonext_16_glue node:$ptr)>; -def atomic_load_nonext_32_#as#_m0 : PatFrag<(ops node:$ptr), - (atomic_load_nonext_32_glue node:$ptr)>; -def atomic_load_nonext_64_#as#_m0 : PatFrag<(ops node:$ptr), - (atomic_load_nonext_64_glue node:$ptr)>; def atomic_load_zext_8_#as#_m0 : PatFrag<(ops node:$ptr), (atomic_load_zext_8_glue node:$ptr)>; @@ -705,44 +691,35 @@ def store_align_less_than_4_#as#_m0 : PatFrag <(ops node:$value, node:$ptr), } } -def atomic_store_8_glue : PatFrag < - (ops node:$ptr, node:$value), - (AMDGPUatomic_st_glue node:$ptr, node:$value)> { - let IsAtomic = 1; - let MemoryVT = i8; -} - -def atomic_store_16_glue : PatFrag < +def atomic_store_glue : PatFrag < (ops node:$ptr, node:$value), (AMDGPUatomic_st_glue node:$ptr, node:$value)> { let IsAtomic = 1; - let MemoryVT = i16; + let IsTruncStore = 0; } -def atomic_store_32_glue : PatFrag < +def atomic_store_8_glue : PatFrag < (ops node:$ptr, node:$value), (AMDGPUatomic_st_glue node:$ptr, node:$value)> { let IsAtomic = 1; - let MemoryVT = i32; + let MemoryVT = i8; } -def atomic_store_64_glue : PatFrag < +def atomic_store_16_glue : PatFrag < (ops node:$ptr, node:$value), (AMDGPUatomic_st_glue node:$ptr, node:$value)> { let IsAtomic = 1; - let MemoryVT = i64; + let MemoryVT = i16; } foreach as = [ "local", "region" ] in let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in { +def atomic_store_#as#_m0 : PatFrag<(ops node:$val, node:$ptr), + (atomic_store_glue node:$val, node:$ptr)>; def atomic_store_8_#as#_m0 : PatFrag<(ops node:$val, node:$ptr), (atomic_store_8_glue node:$val, node:$ptr)>; def atomic_store_16_#as#_m0 : PatFrag<(ops node:$val, node:$ptr), (atomic_store_16_glue node:$val, node:$ptr)>; -def atomic_store_32_#as#_m0 : PatFrag<(ops node:$val, node:$ptr), - (atomic_store_32_glue node:$val, node:$ptr)>; -def atomic_store_64_#as#_m0 : PatFrag<(ops node:$val, node:$ptr), - (atomic_store_64_glue node:$val, node:$ptr)>; } // End let IsAtomic = 1, AddressSpaces = StoreAddress_#as.AddrSpaces diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir index 1dee03e4627b4..984184a9dd54d 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir @@ -82,16 +82,26 @@ body: | ; GFX6-LABEL: name: load_atomic_global_v2s16_seq_cst ; GFX6: liveins: $vgpr0_vgpr1 ; GFX6-NEXT: {{ $}} - ; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX6-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1) - ; GFX6-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX6-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX6-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0 + ; GFX6-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440 + ; GFX6-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1 + ; GFX6-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0 + ; GFX6-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3 + ; GFX6-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1) + ; GFX6-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]] ; ; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst ; GFX7: liveins: $vgpr0_vgpr1 ; GFX7-NEXT: {{ $}} ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1) - ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] + ; GFX7-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0 + ; GFX7-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 61440 + ; GFX7-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1 + ; GFX7-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0 + ; GFX7-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE]], %subreg.sub2_sub3 + ; GFX7-NEXT: [[BUFFER_LOAD_DWORD_ADDR64_:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 [[COPY]], [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1) + ; GFX7-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_ADDR64_]] ; ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/229688 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
