https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/238840
>From 4e3f237d67e462dd5504853ab03df809a43da87e Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Wed, 4 Nov 2026 16:24:18 +0100 Subject: [PATCH 1/3] [AMDGPU] Price constant address space memory operations Price constant address space accesses as global accesses, except for loads selected to scalar loads of up to 64 bytes. These are loads from the 32-bit constant address space, and loads from the constant address space whose pointer is an argument passed in SGPRs or a constant, plus constant offsets, both aligned to 4 bytes or to their size. A scalar load is widened as its lowering does. An integer whose size is not a multiple of 4 bytes is loaded as whole dwords, plus a byte load when its size is 7 modulo 8, since the last byte of such a remainder is left out of the scalar load. A type with no legal dword vector equivalent is split into the largest power of 2 pieces that fit, and once the remainder fits in the alignment it is widened to a power of 2 piece of up to the alignment, at most 8 bytes for byte vectors. A 3-dword remainder of a legal dword vector takes a fourth dword when aligned to 8 on a subtarget without scalar dwordx3 loads. --- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 60 +++++- llvm/test/Analysis/CostModel/AMDGPU/load.ll | 199 ++++++++++++++---- .../AMDGPU/ordered-reduction-fma-fusion.ll | 192 ++++++++++------- 3 files changed, 339 insertions(+), 112 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index 544c8914669f98..21d6770b35f270 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -2172,13 +2172,45 @@ static unsigned getNumMemoryInsts(unsigned Bytes, unsigned MaxBytes, bool HasDwordx3) { unsigned NumInsts = Bytes / MaxBytes; unsigned Rem = Bytes % MaxBytes; - if (HasDwordx3 && Rem >= 12) { + if (HasDwordx3 && Rem % 16 >= 12) { ++NumInsts; Rem -= 12; } return NumInsts + llvm::popcount(Rem); } +static unsigned getNumWidenedSMEMLoads(unsigned Bytes, uint64_t Alignment, + unsigned EltBytes, unsigned MaxPiece, + bool HasDwordx3) { + unsigned WideBytes = PowerOf2Ceil(Bytes); + unsigned NumX16 = 0, SmallBytes = 0; + for (unsigned Off = 0; Off < Bytes;) { + unsigned Rem = Bytes - Off; + unsigned Span = Rem; + if (Rem <= Alignment) + Span = std::min<uint64_t>(Alignment, WideBytes - Off); + unsigned Piece = std::min(llvm::bit_floor(Span), MaxPiece); + if (Rem <= EltBytes) + Piece = 4; + if (Piece >= 64) + NumX16 += divideCeil(std::min(Piece, Rem), 64); + else + SmallBytes += std::max(Piece, 4u); + Off += Piece; + } + return NumX16 + getNumMemoryInsts(SmallBytes, 32, HasDwordx3); +} + +static bool hasUniformPointerOperand(const Instruction *I) { + const auto *LI = dyn_cast_or_null<LoadInst>(I); + if (!LI) + return false; + const Value *Ptr = LI->getPointerOperand()->stripInBoundsConstantOffsets(); + if (const auto *Arg = dyn_cast<Argument>(Ptr)) + return AMDGPU::isArgPassedInSGPR(Arg); + return isa<Constant>(Ptr); +} + InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, @@ -2194,9 +2226,7 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned EltBits = std::max<unsigned>(8, DL.getTypeSizeInBits(EltTy)); - if (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS || - AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT || - AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) { + if (AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) { if (Src->isVectorTy() && EltTy->isIntegerTy(8)) return divideCeil(DL.getTypeSizeInBits(Src) - 1, getLoadStoreVecRegBitWidth(AddressSpace)); @@ -2214,6 +2244,28 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, } unsigned Bytes = DL.getTypeStoreSize(Src); + if (Opcode == Instruction::Load && + (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS || + AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && + Alignment >= std::min(Bytes, 4u) && + (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT || + hasUniformPointerOperand(I))) { + unsigned Dwords = divideCeil(Bytes, 4); + bool HasDwordx3 = ST->hasScalarDwordx3Loads(); + // The last byte of a 7-byte remainder is not widened into the scalar load. + if (!VecTy && Bytes % 4) + return getNumMemoryInsts(Dwords * 4, 32, HasDwordx3) + (Bytes % 8 == 7); + if (Bytes % 4 || !TLI->isTypeLegal(EVT::getVectorVT(Src->getContext(), + MVT::i32, Dwords))) { + unsigned EltBytes = Bytes % 4 ? EltBits / 8 : 4; + return getNumWidenedSMEMLoads(Bytes, Alignment.value(), EltBytes, + EltBytes == 1 ? 8 : 128, HasDwordx3); + } + if (!HasDwordx3 && Alignment >= 8 && Dwords % 4 == 3) + ++Dwords; + return getNumMemoryInsts(Dwords * 4, 64, HasDwordx3); + } + unsigned MaxBytes = std::min(16u, getLoadStoreVecRegBitWidth(AddressSpace) / 8); if (AddressSpace == AMDGPUAS::FLAT_ADDRESS && diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll index b0fc6e119cfa0c..91d9bd63d2f793 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll @@ -913,21 +913,69 @@ define void @loads_odd_widths(i32 %arg) { } define amdgpu_kernel void @loads_addrspace_4_kernarg(ptr addrspace(4) %p, ptr addrspace(6) %q) { -; CHECK-LABEL: 'loads_addrspace_4_kernarg' -; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = load <3 x i16>, ptr addrspace(4) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(4) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(4) %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <16 x i32>, ptr addrspace(4) %p, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <32 x i32>, ptr addrspace(4) %p, align 128 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; GFX8-LABEL: 'loads_addrspace_4_kernarg' +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i16>, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(4) %p, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(4) %p, align 16 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <16 x i32>, ptr addrspace(4) %p, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <32 x i32>, ptr addrspace(4) %p, align 128 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 +; GFX8-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX90A-UNALIGNED-LABEL: 'loads_addrspace_4_kernarg' +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i16>, ptr addrspace(4) %p, align 8 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(4) %p, align 4 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(4) %p, align 16 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <16 x i32>, ptr addrspace(4) %p, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <32 x i32>, ptr addrspace(4) %p, align 128 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX90A-NOUNALIGNED-LABEL: 'loads_addrspace_4_kernarg' +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i16>, ptr addrspace(4) %p, align 8 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(4) %p, align 4 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(4) %p, align 16 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <16 x i32>, ptr addrspace(4) %p, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <32 x i32>, ptr addrspace(4) %p, align 128 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX12-LABEL: 'loads_addrspace_4_kernarg' +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i16>, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(4) %p, align 4 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(4) %p, align 16 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <16 x i32>, ptr addrspace(4) %p, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <32 x i32>, ptr addrspace(4) %p, align 128 +; GFX12-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 +; GFX12-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id +; GFX12-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; load <3 x i16>, ptr addrspace(4) %p, align 8 load <3 x i32>, ptr addrspace(4) %p, align 4 @@ -945,9 +993,42 @@ define amdgpu_kernel void @loads_addrspace_4_kernarg(ptr addrspace(4) %p, ptr ad ret void } +define amdgpu_kernel void @loads_addrspace_4_widened(ptr addrspace(4) %p) { +; GFX8-LABEL: 'loads_addrspace_4_widened' +; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = load <7 x i32>, ptr addrspace(4) %p, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <7 x i32>, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %3 = load <7 x i64>, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <7 x i64>, ptr addrspace(4) %p, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i96, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX90A-LABEL: 'loads_addrspace_4_widened' +; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = load <7 x i32>, ptr addrspace(4) %p, align 4 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <7 x i32>, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %3 = load <7 x i64>, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <7 x i64>, ptr addrspace(4) %p, align 64 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i96, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX12-LABEL: 'loads_addrspace_4_widened' +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load <7 x i32>, ptr addrspace(4) %p, align 4 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <7 x i32>, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %3 = load <7 x i64>, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <7 x i64>, ptr addrspace(4) %p, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i96, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; + load <7 x i32>, ptr addrspace(4) %p, align 4 + load <7 x i32>, ptr addrspace(4) %p, align 8 + load <7 x i64>, ptr addrspace(4) %p, align 8 + load <7 x i64>, ptr addrspace(4) %p, align 64 + load i96, ptr addrspace(4) %p, align 8 + ret void +} + define void @loads_addrspace_4_function_arg(ptr addrspace(4) %p, ptr addrspace(4) inreg %q, ptr addrspace(6) %r) { ; CHECK-LABEL: 'loads_addrspace_4_function_arg' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <16 x i32>, ptr addrspace(4) %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <16 x i32>, ptr addrspace(4) %p, align 64 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i32>, ptr addrspace(4) %q, align 64 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <16 x i32>, ptr addrspace(6) %r, align 64 ; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void @@ -959,27 +1040,71 @@ define void @loads_addrspace_4_function_arg(ptr addrspace(4) %p, ptr addrspace(4 } define amdgpu_kernel void @loads_addrspace_4_illegal_types(ptr addrspace(4) %p) { -; CHECK-LABEL: 'loads_addrspace_4_illegal_types' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <15 x i32>, ptr addrspace(4) %p, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <15 x i32>, ptr addrspace(4) %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <31 x i32>, ptr addrspace(4) %p, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <62 x i32>, ptr addrspace(4) %p, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <7 x i64>, ptr addrspace(4) %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <60 x i8>, ptr addrspace(4) %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <9 x i8>, ptr addrspace(4) %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <11 x i8>, ptr addrspace(4) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <5 x i16>, ptr addrspace(4) %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %10 = load <31 x i16>, ptr addrspace(4) %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load i56, ptr addrspace(4) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load i80, ptr addrspace(4) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %13 = load i80, ptr addrspace(4) %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = load i88, ptr addrspace(4) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = load i120, ptr addrspace(4) %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %16 = load i136, ptr addrspace(4) %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %17 = load i184, ptr addrspace(4) %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %18 = load i248, ptr addrspace(4) %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %19 = load i416, ptr addrspace(4) %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; GFX8-LABEL: 'loads_addrspace_4_illegal_types' +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <15 x i32>, ptr addrspace(4) %p, align 4 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <15 x i32>, ptr addrspace(4) %p, align 16 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <31 x i32>, ptr addrspace(4) %p, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <62 x i32>, ptr addrspace(4) %p, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <7 x i64>, ptr addrspace(4) %p, align 32 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <60 x i8>, ptr addrspace(4) %p, align 32 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <9 x i8>, ptr addrspace(4) %p, align 16 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <11 x i8>, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <5 x i16>, ptr addrspace(4) %p, align 16 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load <31 x i16>, ptr addrspace(4) %p, align 32 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load i56, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load i80, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %13 = load i80, ptr addrspace(4) %p, align 16 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = load i88, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = load i120, ptr addrspace(4) %p, align 16 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = load i136, ptr addrspace(4) %p, align 32 +; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %17 = load i184, ptr addrspace(4) %p, align 8 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load i248, ptr addrspace(4) %p, align 16 +; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = load i416, ptr addrspace(4) %p, align 32 +; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX90A-LABEL: 'loads_addrspace_4_illegal_types' +; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <15 x i32>, ptr addrspace(4) %p, align 4 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <15 x i32>, ptr addrspace(4) %p, align 16 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <31 x i32>, ptr addrspace(4) %p, align 64 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <62 x i32>, ptr addrspace(4) %p, align 64 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <7 x i64>, ptr addrspace(4) %p, align 32 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <60 x i8>, ptr addrspace(4) %p, align 32 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <9 x i8>, ptr addrspace(4) %p, align 16 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <11 x i8>, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <5 x i16>, ptr addrspace(4) %p, align 16 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load <31 x i16>, ptr addrspace(4) %p, align 32 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load i56, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load i80, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %13 = load i80, ptr addrspace(4) %p, align 16 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %14 = load i88, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = load i120, ptr addrspace(4) %p, align 16 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = load i136, ptr addrspace(4) %p, align 32 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %17 = load i184, ptr addrspace(4) %p, align 8 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load i248, ptr addrspace(4) %p, align 16 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = load i416, ptr addrspace(4) %p, align 32 +; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void +; +; GFX12-LABEL: 'loads_addrspace_4_illegal_types' +; GFX12-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = load <15 x i32>, ptr addrspace(4) %p, align 4 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <15 x i32>, ptr addrspace(4) %p, align 16 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <31 x i32>, ptr addrspace(4) %p, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <62 x i32>, ptr addrspace(4) %p, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <7 x i64>, ptr addrspace(4) %p, align 32 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <60 x i8>, ptr addrspace(4) %p, align 32 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <9 x i8>, ptr addrspace(4) %p, align 16 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <11 x i8>, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <5 x i16>, ptr addrspace(4) %p, align 16 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load <31 x i16>, ptr addrspace(4) %p, align 32 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load i56, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load i80, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i80, ptr addrspace(4) %p, align 16 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load i88, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %15 = load i120, ptr addrspace(4) %p, align 16 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = load i136, ptr addrspace(4) %p, align 32 +; GFX12-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %17 = load i184, ptr addrspace(4) %p, align 8 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load i248, ptr addrspace(4) %p, align 16 +; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = load i416, ptr addrspace(4) %p, align 32 +; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; load <15 x i32>, ptr addrspace(4) %p, align 4 load <15 x i32>, ptr addrspace(4) %p, align 16 diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll index a25409530b4bb7..489bd72e8c388d 100644 --- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll +++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll @@ -14,98 +14,148 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) { ; CHECK-NEXT: [[MV0:%.*]] = load float, ptr addrspace(4) [[MP1]], align 4 ; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[IV4]], [[MV0]] ; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 1 +; CHECK-NEXT: [[IV1:%.*]] = load float, ptr addrspace(1) [[IP0]], align 4 ; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 1 +; CHECK-NEXT: [[MV1:%.*]] = load float, ptr addrspace(4) [[MP0]], align 4 +; CHECK-NEXT: [[PROD1:%.*]] = fmul contract float [[IV1]], [[MV1]] +; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[PROD1]] +; CHECK-NEXT: [[IP2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 2 +; CHECK-NEXT: [[IV2:%.*]] = load float, ptr addrspace(1) [[IP2]], align 4 +; CHECK-NEXT: [[MP2:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 2 +; CHECK-NEXT: [[MV2:%.*]] = load float, ptr addrspace(4) [[MP2]], align 4 +; CHECK-NEXT: [[PROD2:%.*]] = fmul contract float [[IV2]], [[MV2]] +; CHECK-NEXT: [[ACC2:%.*]] = fadd contract float [[ACC1]], [[PROD2]] +; CHECK-NEXT: [[IP3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 3 +; CHECK-NEXT: [[IV3:%.*]] = load float, ptr addrspace(1) [[IP3]], align 4 +; CHECK-NEXT: [[MP3:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 3 +; CHECK-NEXT: [[MV3:%.*]] = load float, ptr addrspace(4) [[MP3]], align 4 +; CHECK-NEXT: [[PROD3:%.*]] = fmul contract float [[IV3]], [[MV3]] +; CHECK-NEXT: [[ACC3:%.*]] = fadd contract float [[ACC2]], [[PROD3]] +; CHECK-NEXT: [[IP9:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 4 +; CHECK-NEXT: [[IV9:%.*]] = load float, ptr addrspace(1) [[IP9]], align 4 +; CHECK-NEXT: [[MP4:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 4 +; CHECK-NEXT: [[MV4:%.*]] = load float, ptr addrspace(4) [[MP4]], align 4 +; CHECK-NEXT: [[PROD4:%.*]] = fmul contract float [[IV9]], [[MV4]] +; CHECK-NEXT: [[ACC4:%.*]] = fadd contract float [[ACC3]], [[PROD4]] ; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8 +; CHECK-NEXT: [[IV10:%.*]] = load float, ptr addrspace(1) [[IP6]], align 4 +; CHECK-NEXT: [[MP5:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 5 +; CHECK-NEXT: [[MV5:%.*]] = load float, ptr addrspace(4) [[MP5]], align 4 +; CHECK-NEXT: [[PROD5:%.*]] = fmul contract float [[IV10]], [[MV5]] +; CHECK-NEXT: [[ACC5:%.*]] = fadd contract float [[ACC4]], [[PROD5]] +; CHECK-NEXT: [[IP11:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 9 +; CHECK-NEXT: [[IV6:%.*]] = load float, ptr addrspace(1) [[IP11]], align 4 +; CHECK-NEXT: [[MP6:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 6 +; CHECK-NEXT: [[MV6:%.*]] = load float, ptr addrspace(4) [[MP6]], align 4 +; CHECK-NEXT: [[PROD6:%.*]] = fmul contract float [[IV6]], [[MV6]] +; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC5]], [[PROD6]] +; CHECK-NEXT: [[IP7:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 10 +; CHECK-NEXT: [[IV7:%.*]] = load float, ptr addrspace(1) [[IP7]], align 4 +; CHECK-NEXT: [[MP7:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 7 +; CHECK-NEXT: [[MV7:%.*]] = load float, ptr addrspace(4) [[MP7]], align 4 +; CHECK-NEXT: [[PROD7:%.*]] = fmul contract float [[IV7]], [[MV7]] +; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[PROD7]] +; CHECK-NEXT: [[IP13:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 11 +; CHECK-NEXT: [[IV8:%.*]] = load float, ptr addrspace(1) [[IP13]], align 4 +; CHECK-NEXT: [[MP8:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 8 +; CHECK-NEXT: [[MV8:%.*]] = load float, ptr addrspace(4) [[MP8]], align 4 +; CHECK-NEXT: [[PROD8:%.*]] = fmul contract float [[IV8]], [[MV8]] +; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[PROD8]] ; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 12 ; CHECK-NEXT: [[IV5:%.*]] = load float, ptr addrspace(1) [[IP5]], align 4 +; CHECK-NEXT: [[MP9:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 9 +; CHECK-NEXT: [[MV9:%.*]] = load float, ptr addrspace(4) [[MP9]], align 4 +; CHECK-NEXT: [[PROD9:%.*]] = fmul contract float [[IV5]], [[MV9]] +; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[PROD9]] ; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16 ; CHECK-NEXT: [[IV14:%.*]] = load float, ptr addrspace(1) [[IP14]], align 4 +; CHECK-NEXT: [[MP10:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 10 +; CHECK-NEXT: [[MV10:%.*]] = load float, ptr addrspace(4) [[MP10]], align 4 +; CHECK-NEXT: [[PROD10:%.*]] = fmul contract float [[IV14]], [[MV10]] +; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[PROD10]] ; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 17 +; CHECK-NEXT: [[IV11:%.*]] = load float, ptr addrspace(1) [[IP8]], align 4 +; CHECK-NEXT: [[MP11:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 11 +; CHECK-NEXT: [[MV11:%.*]] = load float, ptr addrspace(4) [[MP11]], align 4 +; CHECK-NEXT: [[PROD11:%.*]] = fmul contract float [[IV11]], [[MV11]] +; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[PROD11]] +; CHECK-NEXT: [[IP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 18 +; CHECK-NEXT: [[IV12:%.*]] = load float, ptr addrspace(1) [[IP15]], align 4 +; CHECK-NEXT: [[MP12:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 12 +; CHECK-NEXT: [[MV12:%.*]] = load float, ptr addrspace(4) [[MP12]], align 4 +; CHECK-NEXT: [[PROD12:%.*]] = fmul contract float [[IV12]], [[MV12]] +; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[PROD12]] ; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 19 +; CHECK-NEXT: [[IV13:%.*]] = load float, ptr addrspace(1) [[IP10]], align 4 +; CHECK-NEXT: [[MP13:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 13 +; CHECK-NEXT: [[MV13:%.*]] = load float, ptr addrspace(4) [[MP13]], align 4 +; CHECK-NEXT: [[PROD13:%.*]] = fmul contract float [[IV13]], [[MV13]] +; CHECK-NEXT: [[ACC14:%.*]] = fadd contract float [[ACC12]], [[PROD13]] +; CHECK-NEXT: [[IP25:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 20 +; CHECK-NEXT: [[IV25:%.*]] = load float, ptr addrspace(1) [[IP25]], align 4 +; CHECK-NEXT: [[MP14:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 14 +; CHECK-NEXT: [[MV14:%.*]] = load float, ptr addrspace(4) [[MP14]], align 4 +; CHECK-NEXT: [[PROD14:%.*]] = fmul contract float [[IV25]], [[MV14]] +; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC14]], [[PROD14]] ; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24 -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[IP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[IP6]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[IP12]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = load <16 x float>, ptr addrspace(4) [[MP0]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV5]], i64 8 -; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV14]], i64 9 -; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison> -; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> [[TMP11]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> -; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison> -; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> -; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison> -; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x float> [[TMP16]], <16 x float> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15> -; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison> -; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15> -; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison> -; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP20]], <16 x float> [[TMP22]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17> -; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP24]], [[TMP6]] -; CHECK-NEXT: [[ACC14:%.*]] = extractelement <16 x float> [[TMP21]], i64 0 -; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[ACC14]] -; CHECK-NEXT: [[PROD15:%.*]] = extractelement <16 x float> [[TMP21]], i64 1 -; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC1]], [[PROD15]] -; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP21]], i64 2 -; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC15]], [[TMP9]] -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP21]], i64 3 -; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC16]], [[TMP10]] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP21]], i64 4 -; CHECK-NEXT: [[ACC18:%.*]] = fadd contract float [[ACC17]], [[TMP14]] -; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP21]], i64 5 -; CHECK-NEXT: [[ACC19:%.*]] = fadd contract float [[ACC18]], [[TMP15]] -; CHECK-NEXT: [[TMP28:%.*]] = extractelement <16 x float> [[TMP21]], i64 6 -; CHECK-NEXT: [[ACC6:%.*]] = fadd contract float [[ACC19]], [[TMP28]] -; CHECK-NEXT: [[TMP29:%.*]] = extractelement <16 x float> [[TMP21]], i64 7 -; CHECK-NEXT: [[ACC7:%.*]] = fadd contract float [[ACC6]], [[TMP29]] -; CHECK-NEXT: [[TMP30:%.*]] = extractelement <16 x float> [[TMP21]], i64 8 -; CHECK-NEXT: [[ACC8:%.*]] = fadd contract float [[ACC7]], [[TMP30]] -; CHECK-NEXT: [[TMP31:%.*]] = extractelement <16 x float> [[TMP21]], i64 9 -; CHECK-NEXT: [[ACC9:%.*]] = fadd contract float [[ACC8]], [[TMP31]] -; CHECK-NEXT: [[TMP32:%.*]] = extractelement <16 x float> [[TMP21]], i64 10 -; CHECK-NEXT: [[ACC10:%.*]] = fadd contract float [[ACC9]], [[TMP32]] -; CHECK-NEXT: [[TMP33:%.*]] = extractelement <16 x float> [[TMP21]], i64 11 -; CHECK-NEXT: [[ACC11:%.*]] = fadd contract float [[ACC10]], [[TMP33]] -; CHECK-NEXT: [[TMP34:%.*]] = extractelement <16 x float> [[TMP21]], i64 12 -; CHECK-NEXT: [[ACC12:%.*]] = fadd contract float [[ACC11]], [[TMP34]] -; CHECK-NEXT: [[TMP35:%.*]] = extractelement <16 x float> [[TMP21]], i64 13 -; CHECK-NEXT: [[ACC13:%.*]] = fadd contract float [[ACC12]], [[TMP35]] -; CHECK-NEXT: [[TMP36:%.*]] = extractelement <16 x float> [[TMP21]], i64 14 +; CHECK-NEXT: [[IV15:%.*]] = load float, ptr addrspace(1) [[IP12]], align 4 +; CHECK-NEXT: [[MP15:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 15 +; CHECK-NEXT: [[MV15:%.*]] = load float, ptr addrspace(4) [[MP15]], align 4 +; CHECK-NEXT: [[TMP36:%.*]] = fmul contract float [[IV15]], [[MV15]] ; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP36]] -; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 15 +; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25 +; CHECK-NEXT: [[IV16:%.*]] = load float, ptr addrspace(1) [[IP16]], align 4 +; CHECK-NEXT: [[MP17:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16 +; CHECK-NEXT: [[MV16:%.*]] = load float, ptr addrspace(4) [[MP17]], align 4 +; CHECK-NEXT: [[TMP37:%.*]] = fmul contract float [[IV16]], [[MV16]] ; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]] ; CHECK-NEXT: [[IP17:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 26 +; CHECK-NEXT: [[IV17:%.*]] = load float, ptr addrspace(1) [[IP17]], align 4 ; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 17 +; CHECK-NEXT: [[MV17:%.*]] = load float, ptr addrspace(4) [[MP16]], align 4 +; CHECK-NEXT: [[PROD17:%.*]] = fmul contract float [[IV17]], [[MV17]] +; CHECK-NEXT: [[ACC17:%.*]] = fadd contract float [[ACC26]], [[PROD17]] +; CHECK-NEXT: [[IP18:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 27 +; CHECK-NEXT: [[IV18:%.*]] = load float, ptr addrspace(1) [[IP18]], align 4 +; CHECK-NEXT: [[MP18:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 18 +; CHECK-NEXT: [[MV18:%.*]] = load float, ptr addrspace(4) [[MP18]], align 4 +; CHECK-NEXT: [[PROD18:%.*]] = fmul contract float [[IV18]], [[MV18]] +; CHECK-NEXT: [[ACC18:%.*]] = fadd contract float [[ACC17]], [[PROD18]] ; CHECK-NEXT: [[IP19:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 28 ; CHECK-NEXT: [[IV19:%.*]] = load float, ptr addrspace(1) [[IP19]], align 4 +; CHECK-NEXT: [[MP19:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 19 +; CHECK-NEXT: [[MV19:%.*]] = load float, ptr addrspace(4) [[MP19]], align 4 +; CHECK-NEXT: [[PROD19:%.*]] = fmul contract float [[IV19]], [[MV19]] +; CHECK-NEXT: [[ACC19:%.*]] = fadd contract float [[ACC18]], [[PROD19]] ; CHECK-NEXT: [[IP20:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 32 ; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP20]], align 4 +; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 20 +; CHECK-NEXT: [[MV20:%.*]] = load float, ptr addrspace(4) [[MP20]], align 4 +; CHECK-NEXT: [[PROD20:%.*]] = fmul contract float [[IV20]], [[MV20]] +; CHECK-NEXT: [[ACC30:%.*]] = fadd contract float [[ACC19]], [[PROD20]] ; CHECK-NEXT: [[IP21:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 33 -; CHECK-NEXT: [[TMP38:%.*]] = load <2 x float>, ptr addrspace(1) [[IP17]], align 4 -; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP21]], align 4 -; CHECK-NEXT: [[TMP40:%.*]] = load <8 x float>, ptr addrspace(4) [[MP16]], align 4 -; CHECK-NEXT: [[TMP41:%.*]] = insertelement <8 x float> poison, float [[IV19]], i64 2 -; CHECK-NEXT: [[TMP53:%.*]] = insertelement <8 x float> [[TMP41]], float [[IV20]], i64 3 -; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x float> [[TMP39]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison> -; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <8 x float> [[TMP53]], <8 x float> [[TMP42]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11> -; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <2 x float> [[TMP38]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison> -; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <8 x float> [[TMP54]], <8 x float> [[TMP55]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> -; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <8 x float> [[TMP43]], [[TMP40]] -; CHECK-NEXT: [[TMP45:%.*]] = extractelement <8 x float> [[TMP44]], i64 0 -; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC26]], [[TMP45]] -; CHECK-NEXT: [[TMP46:%.*]] = extractelement <8 x float> [[TMP44]], i64 1 -; CHECK-NEXT: [[ACC28:%.*]] = fadd contract float [[ACC27]], [[TMP46]] -; CHECK-NEXT: [[TMP47:%.*]] = extractelement <8 x float> [[TMP44]], i64 2 -; CHECK-NEXT: [[ACC29:%.*]] = fadd contract float [[ACC28]], [[TMP47]] -; CHECK-NEXT: [[TMP48:%.*]] = extractelement <8 x float> [[TMP44]], i64 3 -; CHECK-NEXT: [[ACC30:%.*]] = fadd contract float [[ACC29]], [[TMP48]] -; CHECK-NEXT: [[TMP49:%.*]] = extractelement <8 x float> [[TMP44]], i64 4 +; CHECK-NEXT: [[IV21:%.*]] = load float, ptr addrspace(1) [[IP21]], align 4 +; CHECK-NEXT: [[MP21:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 21 +; CHECK-NEXT: [[MV21:%.*]] = load float, ptr addrspace(4) [[MP21]], align 4 +; CHECK-NEXT: [[TMP49:%.*]] = fmul contract float [[IV21]], [[MV21]] ; CHECK-NEXT: [[ACC20:%.*]] = fadd contract float [[ACC30]], [[TMP49]] -; CHECK-NEXT: [[TMP50:%.*]] = extractelement <8 x float> [[TMP44]], i64 5 +; CHECK-NEXT: [[IP22:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 34 +; CHECK-NEXT: [[IV22:%.*]] = load float, ptr addrspace(1) [[IP22]], align 4 +; CHECK-NEXT: [[MP22:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 22 +; CHECK-NEXT: [[MV22:%.*]] = load float, ptr addrspace(4) [[MP22]], align 4 +; CHECK-NEXT: [[TMP50:%.*]] = fmul contract float [[IV22]], [[MV22]] ; CHECK-NEXT: [[ACC21:%.*]] = fadd contract float [[ACC20]], [[TMP50]] -; CHECK-NEXT: [[TMP51:%.*]] = extractelement <8 x float> [[TMP44]], i64 6 +; CHECK-NEXT: [[IP23:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 35 +; CHECK-NEXT: [[IV23:%.*]] = load float, ptr addrspace(1) [[IP23]], align 4 +; CHECK-NEXT: [[MP23:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 23 +; CHECK-NEXT: [[MV23:%.*]] = load float, ptr addrspace(4) [[MP23]], align 4 +; CHECK-NEXT: [[TMP51:%.*]] = fmul contract float [[IV23]], [[MV23]] ; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP51]] -; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x float> [[TMP44]], i64 7 +; CHECK-NEXT: [[IP24:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 36 +; CHECK-NEXT: [[IV24:%.*]] = load float, ptr addrspace(1) [[IP24]], align 4 +; CHECK-NEXT: [[MP24:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 24 +; CHECK-NEXT: [[MV24:%.*]] = load float, ptr addrspace(4) [[MP24]], align 4 +; CHECK-NEXT: [[TMP52:%.*]] = fmul contract float [[IV24]], [[MV24]] ; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC22]], [[TMP52]] ; CHECK-NEXT: ret float [[ACC24]] ; >From d7f70168347f215ab75717556cc669205cf3e924 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Wed, 4 Nov 2026 17:57:10 +0100 Subject: [PATCH 2/3] small changes --- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 26 ++++++++------ llvm/test/Analysis/CostModel/AMDGPU/load.ll | 34 ++++++++++++------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index 21d6770b35f270..3bed0c65bf652a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -2180,10 +2180,11 @@ static unsigned getNumMemoryInsts(unsigned Bytes, unsigned MaxBytes, } static unsigned getNumWidenedSMEMLoads(unsigned Bytes, uint64_t Alignment, - unsigned EltBytes, unsigned MaxPiece, + unsigned EltBytes, unsigned MaxBytes, bool HasDwordx3) { + unsigned MaxPiece = EltBytes == 1 ? 8 : MaxBytes; unsigned WideBytes = PowerOf2Ceil(Bytes); - unsigned NumX16 = 0, SmallBytes = 0; + unsigned NumWide = 0, SmallBytes = 0; for (unsigned Off = 0; Off < Bytes;) { unsigned Rem = Bytes - Off; unsigned Span = Rem; @@ -2192,20 +2193,24 @@ static unsigned getNumWidenedSMEMLoads(unsigned Bytes, uint64_t Alignment, unsigned Piece = std::min(llvm::bit_floor(Span), MaxPiece); if (Rem <= EltBytes) Piece = 4; - if (Piece >= 64) - NumX16 += divideCeil(std::min(Piece, Rem), 64); + if (Piece >= MaxBytes) + ++NumWide; else SmallBytes += std::max(Piece, 4u); Off += Piece; } - return NumX16 + getNumMemoryInsts(SmallBytes, 32, HasDwordx3); + return NumWide + getNumMemoryInsts(SmallBytes, 32, HasDwordx3); } -static bool hasUniformPointerOperand(const Instruction *I) { +static bool hasUniformPointerOperand(const Instruction *I, + const DataLayout &DL) { const auto *LI = dyn_cast_or_null<LoadInst>(I); if (!LI) return false; - const Value *Ptr = LI->getPointerOperand()->stripInBoundsConstantOffsets(); + const Value *Ptr = LI->getPointerOperand(); + APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); + Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset, + /*AllowNonInbounds=*/true); if (const auto *Arg = dyn_cast<Argument>(Ptr)) return AMDGPU::isArgPassedInSGPR(Arg); return isa<Constant>(Ptr); @@ -2249,8 +2254,9 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && Alignment >= std::min(Bytes, 4u) && (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT || - hasUniformPointerOperand(I))) { + hasUniformPointerOperand(I, DL))) { unsigned Dwords = divideCeil(Bytes, 4); + unsigned MaxBytes = getLoadStoreVecRegBitWidth(AddressSpace) / 8; bool HasDwordx3 = ST->hasScalarDwordx3Loads(); // The last byte of a 7-byte remainder is not widened into the scalar load. if (!VecTy && Bytes % 4) @@ -2259,11 +2265,11 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, MVT::i32, Dwords))) { unsigned EltBytes = Bytes % 4 ? EltBits / 8 : 4; return getNumWidenedSMEMLoads(Bytes, Alignment.value(), EltBytes, - EltBytes == 1 ? 8 : 128, HasDwordx3); + MaxBytes, HasDwordx3); } if (!HasDwordx3 && Alignment >= 8 && Dwords % 4 == 3) ++Dwords; - return getNumMemoryInsts(Dwords * 4, 64, HasDwordx3); + return getNumMemoryInsts(Dwords * 4, MaxBytes, HasDwordx3); } unsigned MaxBytes = diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll index 91d9bd63d2f793..6d53a9454a975a 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll @@ -922,11 +922,13 @@ define amdgpu_kernel void @loads_addrspace_4_kernarg(ptr addrspace(4) %p, ptr ad ; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 ; GFX8-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 -; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep.raw = getelementptr i8, ptr addrspace(4) %p, i64 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(4) %gep.raw, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <16 x i32>, ptr addrspace(4) @g, align 64 ; GFX8-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() ; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id -; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %11 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 ; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX90A-UNALIGNED-LABEL: 'loads_addrspace_4_kernarg' @@ -938,11 +940,13 @@ define amdgpu_kernel void @loads_addrspace_4_kernarg(ptr addrspace(4) %p, ptr ad ; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 ; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 ; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 -; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 -; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep.raw = getelementptr i8, ptr addrspace(4) %p, i64 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(4) %gep.raw, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <16 x i32>, ptr addrspace(4) @g, align 64 ; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() ; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id -; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %11 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 ; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX90A-NOUNALIGNED-LABEL: 'loads_addrspace_4_kernarg' @@ -954,11 +958,13 @@ define amdgpu_kernel void @loads_addrspace_4_kernarg(ptr addrspace(4) %p, ptr ad ; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 ; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 ; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 -; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 -; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep.raw = getelementptr i8, ptr addrspace(4) %p, i64 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(4) %gep.raw, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <16 x i32>, ptr addrspace(4) @g, align 64 ; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() ; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id -; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %11 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 ; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX12-LABEL: 'loads_addrspace_4_kernarg' @@ -970,11 +976,13 @@ define amdgpu_kernel void @loads_addrspace_4_kernarg(ptr addrspace(4) %p, ptr ad ; GFX12-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <16 x i32>, ptr addrspace(4) %p, align 2 ; GFX12-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 ; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <16 x i32>, ptr addrspace(4) %gep, align 64 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(6) %q, align 64 -; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(4) @g, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %gep.raw = getelementptr i8, ptr addrspace(4) %p, i64 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <16 x i32>, ptr addrspace(4) %gep.raw, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <16 x i32>, ptr addrspace(6) %q, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <16 x i32>, ptr addrspace(4) @g, align 64 ; GFX12-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %id = call i32 @llvm.amdgcn.workitem.id.x() ; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %gep.id = getelementptr inbounds <16 x i32>, ptr addrspace(4) %p, i32 %id -; GFX12-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 +; GFX12-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %11 = load <16 x i32>, ptr addrspace(4) %gep.id, align 64 ; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; load <3 x i16>, ptr addrspace(4) %p, align 8 @@ -985,6 +993,8 @@ define amdgpu_kernel void @loads_addrspace_4_kernarg(ptr addrspace(4) %p, ptr ad load <16 x i32>, ptr addrspace(4) %p, align 2 %gep = getelementptr inbounds i8, ptr addrspace(4) %p, i64 64 load <16 x i32>, ptr addrspace(4) %gep, align 64 + %gep.raw = getelementptr i8, ptr addrspace(4) %p, i64 64 + load <16 x i32>, ptr addrspace(4) %gep.raw, align 64 load <16 x i32>, ptr addrspace(6) %q, align 64 load <16 x i32>, ptr addrspace(4) @g, align 64 %id = call i32 @llvm.amdgcn.workitem.id.x() >From be9ab7cc75ed9a98aef59109451c945c6dea45db Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Wed, 4 Nov 2026 22:16:32 +0100 Subject: [PATCH 3/3] Use isSourceOfDivergence in hasUniformPointerOperand --- llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp | 9 +++------ llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h | 2 ++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index 3bed0c65bf652a..f390226ef7297d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -2202,8 +2202,7 @@ static unsigned getNumWidenedSMEMLoads(unsigned Bytes, uint64_t Alignment, return NumWide + getNumMemoryInsts(SmallBytes, 32, HasDwordx3); } -static bool hasUniformPointerOperand(const Instruction *I, - const DataLayout &DL) { +bool GCNTTIImpl::hasUniformPointerOperand(const Instruction *I) const { const auto *LI = dyn_cast_or_null<LoadInst>(I); if (!LI) return false; @@ -2211,9 +2210,7 @@ static bool hasUniformPointerOperand(const Instruction *I, APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset, /*AllowNonInbounds=*/true); - if (const auto *Arg = dyn_cast<Argument>(Ptr)) - return AMDGPU::isArgPassedInSGPR(Arg); - return isa<Constant>(Ptr); + return isa<Argument, Constant>(Ptr) && !isSourceOfDivergence(Ptr); } InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, @@ -2254,7 +2251,7 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && Alignment >= std::min(Bytes, 4u) && (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT || - hasUniformPointerOperand(I, DL))) { + hasUniformPointerOperand(I))) { unsigned Dwords = divideCeil(Bytes, 4); unsigned MaxBytes = getLoadStoreVecRegBitWidth(AddressSpace) / 8; bool HasDwordx3 = ST->hasScalarDwordx3Loads(); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h index ae218097e990b4..d0d2a379e3b06c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h @@ -110,6 +110,8 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> { /// uniform result even taking non-uniform arguments. bool isAlwaysUniform(const Value *V) const; + bool hasUniformPointerOperand(const Instruction *I) const; + public: explicit GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F); _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
