https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/214248
>From 83b9548c82c4b6ca1a5a5e5625ba7fe87e8e2efa Mon Sep 17 00:00:00 2001 From: Gheorghe-Teodor Bercea <[email protected]> Date: Wed, 5 Aug 2026 17:57:28 +0300 Subject: [PATCH] Allocate and reserve registers for address space 13 objects --- llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 9 + llvm/lib/Target/AMDGPU/AMDGPU.h | 23 +- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 34 +- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 2 + llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 15 +- llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h | 19 + llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp | 26 ++ llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h | 19 + llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 2 + .../AMDGPU/AMDGPUPrivateObjectVGPRs.cpp | 340 ++++++++++++++++++ .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 152 +++++++- .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 23 +- llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 56 +++ llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 + llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 7 +- llvm/lib/Target/AMDGPU/SIInstructions.td | 19 + .../AddressSpaceVGPR/as-vgpr-across-call.ll | 70 ++++ .../AddressSpaceVGPR/as-vgpr-allocate.ll | 133 +++++++ .../as-vgpr-object-regalloc.ll | 178 +++++++++ llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 4 + llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 9 +- 22 files changed, 1121 insertions(+), 21 deletions(-) create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll create mode 100644 llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 21882e247c027..1048f99752643 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -4292,4 +4292,13 @@ def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic< /// incoming block. def int_amdgcn_dead: DefaultAttrsIntrinsic<[llvm_any_ty], [], [IntrNoMem]>; + +/// Backend-only intrinsics analogous to llvm.lifetime.{start,end} for marking +/// the lifetimes of allocas in the VGPR address space. +def int_amdgcn_vgpr_lifetime_start : + DefaultAttrsIntrinsic<[], [llvm_anyptr_ty], + [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; +def int_amdgcn_vgpr_lifetime_end : + DefaultAttrsIntrinsic<[], [llvm_anyptr_ty], + [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index 84791f744cf25..4393269ffabc3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -226,6 +226,9 @@ extern char &AMDGPUMarkLastScratchLoadID; void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &); extern char &AMDGPULowerIdxOpsID; +void initializeAMDGPUPrivateObjectVGPRsLegacyPass(PassRegistry &); +extern char &AMDGPUPrivateObjectVGPRsID; + void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &); extern char &SILowerSGPRSpillsLegacyID; @@ -266,7 +269,7 @@ void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &); extern char &AMDGPUPreloadKernelArgumentsLegacyID; // Passes common to R600 and SI -FunctionPass *createAMDGPUPromoteAlloca(); +FunctionPass *createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel); void initializeAMDGPUPromoteAllocaPass(PassRegistry&); extern char &AMDGPUPromoteAllocaID; @@ -279,6 +282,17 @@ struct AMDGPUPromoteAllocaPass TargetMachine &TM; }; +/// Only allocates objects in the VGPR ("as memory") address space, which is +/// required for functionality; used in place of AMDGPUPromoteAllocaPass when +/// optimizations are disabled. +struct AMDGPUVGPRAllocatePass : PassInfoMixin<AMDGPUVGPRAllocatePass> { + AMDGPUVGPRAllocatePass(TargetMachine &TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + +private: + TargetMachine &TM; +}; + struct AMDGPUPromoteAllocaToVectorPass : OptionalPassInfoMixin<AMDGPUPromoteAllocaToVectorPass> { AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {} @@ -447,6 +461,13 @@ class AMDGPULowerIdxOpsPass MachineFunctionAnalysisManager &MFAM); }; +class AMDGPUPrivateObjectVGPRsPass + : public RequiredPassInfoMixin<AMDGPUPrivateObjectVGPRsPass> { +public: + PreservedAnalyses run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM); +}; + class SIInsertWaitcntsPass : public RequiredPassInfoMixin<SIInsertWaitcntsPass> { public: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 0f88afbf20e37..5ba8c02f00b20 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -986,7 +986,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .widenScalarToNextPow2(0, 32) .clampMaxNumElements(0, S32, 16); - getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr}); + // An alloca in the VGPR ("as memory") address space is custom-lowered to the + // address it was allocated; see legalizeFrameIndexVGPR. + getActionDefinitionsBuilder(G_FRAME_INDEX) + .legalFor({PrivatePtr}) + .customFor({VGPRPtr}); // If the amount is divergent, we have to do a wave reduction to get the // maximum value, so this is expanded during RegBankSelect. @@ -2362,6 +2366,8 @@ bool AMDGPULegalizerInfo::legalizeCustom( switch (MI.getOpcode()) { case TargetOpcode::G_ADDRSPACE_CAST: return legalizeAddrSpaceCast(MI, MRI, B); + case TargetOpcode::G_FRAME_INDEX: + return legalizeFrameIndexVGPR(MI, B); case TargetOpcode::G_INTRINSIC_ROUNDEVEN: return legalizeFroundeven(MI, MRI, B); case TargetOpcode::G_FCEIL: @@ -3521,6 +3527,22 @@ static LLT widenToNextPowerOf2(LLT Ty) { /// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for /// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2). /// Parallels the SelectionDAG LowerLoadStoreVGPR. +// The address of an object in the VGPR ("as memory") address space is where +// AMDGPUPromoteAlloca placed it, recorded on the alloca. Parallels the +// SelectionDAG SITargetLowering::lowerFrameIndex. +bool AMDGPULegalizerInfo::legalizeFrameIndexVGPR(MachineInstr &MI, + MachineIRBuilder &B) const { + MachineFunction &MF = B.getMF(); + int FI = MI.getOperand(1).getIndex(); + const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI); + assert(Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR); + + const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca); + B.buildConstant(MI.getOperand(0), MD.getAddress()); + MI.eraseFromParent(); + return true; +} + static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) { MachineIRBuilder &B = Helper.MIRBuilder; MachineRegisterInfo &MRI = *B.getMRI(); @@ -8942,6 +8964,16 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, MI.eraseFromParent(); return true; } + case Intrinsic::amdgcn_vgpr_lifetime_start: + case Intrinsic::amdgcn_vgpr_lifetime_end: { + assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!"); + unsigned Opc = IntrID == Intrinsic::amdgcn_vgpr_lifetime_start + ? AMDGPU::VGPR_LIFETIME_START + : AMDGPU::VGPR_LIFETIME_END; + B.buildInstr(Opc).cloneMemRefs(MI); + MI.eraseFromParent(); + return true; + } case Intrinsic::amdgcn_cooperative_atomic_load_32x4B: case Intrinsic::amdgcn_cooperative_atomic_load_16x8B: case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index 30fd16930e6ae..1cc9119575731 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -41,6 +41,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo { MachineRegisterInfo &MRI, MachineIRBuilder &B) const; + bool legalizeFrameIndexVGPR(MachineInstr &MI, MachineIRBuilder &B) const; + bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp index b109d0f5b2389..c8af1830378dd 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp @@ -16,6 +16,8 @@ #include "AMDGPU.h" #include "AMDGPUAsmPrinter.h" #include "AMDGPUMachineFunctionInfo.h" +#include "AMDGPUMachineInstrs.h" +#include "AMDGPUMemoryUtils.h" #include "MCTargetDesc/AMDGPUInstPrinter.h" #include "MCTargetDesc/AMDGPUMCExpr.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" @@ -435,8 +437,19 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) { } if (MI->isMetaInstruction()) { - if (isVerbose()) + if (isVerbose()) { + if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(MI)) { + const auto &MD = + AMDGPU::AllocatedVGPRsMetadata::get(Marker->getObject()); + unsigned Begin = MD.getAddress() / 4; + unsigned End = (MD.getAddress() + MD.getSize() - 1) / 4; + OutStreamer->emitRawComment( + Twine(" VGPR lifetime ") + (Marker->isStart() ? "start" : "end") + + ": v[" + Twine(Begin) + ":" + Twine(End) + "]"); + return; + } OutStreamer->emitRawComment(" meta instruction"); + } return; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h index e99594062871a..5497eb74de3de 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h @@ -74,6 +74,25 @@ class VStoreIdxInst : public VLoadStoreIdxInst { } }; +// Wrapper for the markers bounding the lifetime of an object in the VGPR "as +// memory" (address space 13) address space. The object is named by the memory +// operand, which is the only thing these carry until +// AMDGPUPrivateObjectVGPRs gives them the registers it occupies. +class VGPRLifetimeInst : public MachineInstr { +public: + bool isStart() const { return getOpcode() == AMDGPU::VGPR_LIFETIME_START; } + + const AllocaInst &getObject() const { + return *cast<AllocaInst>((*memoperands_begin())->getValue()); + } + + static bool classof(const MachineInstr *MI) { + unsigned Opc = MI->getOpcode(); + return Opc == AMDGPU::VGPR_LIFETIME_START || + Opc == AMDGPU::VGPR_LIFETIME_END; + } +}; + } // end namespace AMDGPUMI } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp index 7186ef6a5a827..7614fc23fac40 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp @@ -31,6 +31,32 @@ Align getAlign(const DataLayout &DL, const GlobalVariable *GV) { GV->getValueType()); } +AllocatedVGPRsMetadata &AllocatedVGPRsMetadata::get(const AllocaInst &Alloca) { + return *cast<AllocatedVGPRsMetadata>( + Alloca.getMetadata("amdgpu.allocated.vgprs")); +} + +unsigned AllocatedVGPRsMetadata::getAddress() const { + return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(0))->getValue()) + ->getZExtValue(); +} + +unsigned AllocatedVGPRsMetadata::getSize() const { + return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(1))->getValue()) + ->getZExtValue(); +} + +bool AllocatedVGPRsMetadata::classof(const MDNode *N) { + if (N->getNumOperands() != 2) + return false; + for (int I = 0; I != 2; ++I) { + auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(I)); + if (!C || !isa<ConstantInt>(C->getValue())) + return false; + } + return true; +} + void copyMetadataForWidenedLoad(LoadInst &Dest, const LoadInst &Source) { SmallVector<std::pair<unsigned, MDNode *>, 8> MD; Source.getAllMetadata(MD); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h index f81b677051561..db04c254feaeb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h @@ -12,11 +12,13 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/IR/Metadata.h" namespace llvm { struct Align; class AAResults; +class AllocaInst; class DataLayout; class GlobalVariable; class LoadInst; @@ -106,6 +108,23 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA); bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA, AAResults *AA); +/// Convenience wrapper for !amdgpu.allocated.vgprs metadata, which records +/// where in the VGPR ("as memory") address space an alloca was placed. +/// +/// NOTE: Using `isa` with this class is not very meaningful. +class AllocatedVGPRsMetadata : public MDNode { +public: + static AllocatedVGPRsMetadata &get(const AllocaInst &Alloca); + + /// Get the indicated (byte) address for the VGPR allocation. + unsigned getAddress() const; + + /// Get the size (in bytes) of the VGPR allocation. + unsigned getSize() const; + + static bool classof(const MDNode *N); +}; + } // end namespace AMDGPU } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def index 534d97df145c0..e5385d189b94b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def +++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def @@ -73,6 +73,7 @@ FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass()) FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass()) FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes", AMDGPUUnifyDivergentExitNodesPass()) +FUNCTION_PASS("amdgpu-vgpr-allocate", AMDGPUVGPRAllocatePass(*this)) FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass()) FUNCTION_PASS("si-annotate-control-flow", SIAnnotateControlFlowPass(*static_cast<const GCNTargetMachine *>(this))) FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass()) @@ -124,6 +125,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-agpr-copy-mfma", AMDGPURewriteAGPRCopyMFMA MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass()) MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass()) MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass()) +MACHINE_FUNCTION_PASS("amdgpu-private-object-vgprs", AMDGPUPrivateObjectVGPRsPass()) MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass()) MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass()) MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass()) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp new file mode 100644 index 0000000000000..b4e14795e0708 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp @@ -0,0 +1,340 @@ +//===- AMDGPUPrivateObjectVGPRs.cpp - Private object VGPRs ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// Mark the physical VGPRs an object in the VGPR ("as memory") address space +/// was allocated to as used, so that register allocation does not hand them to +/// anything else where the object is live: +/// +/// * Add implicit use/def operands to the VGPR_LIFETIME_{START,END} pseudos +/// * Add the VGPRs to basic block live-ins +/// +/// The allocation itself is recorded on the alloca as !amdgpu.allocated.vgprs +/// by AMDGPUPromoteAlloca, and reaches this pass through the memory operand of +/// the lifetime markers. +// +//===----------------------------------------------------------------------===// + +#include "AMDGPU.h" +#include "AMDGPUMachineInstrs.h" +#include "AMDGPUMemoryUtils.h" +#include "GCNSubtarget.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/ADT/SmallBitVector.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/LiveVariables.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/Instructions.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/MathExtras.h" + +using namespace llvm; + +#define DEBUG_TYPE "amdgpu-private-object-vgprs" + +using ObjectRegs = SmallVector<MCPhysReg, 50>; + +namespace { + +class AMDGPUPrivateObjectVGPRsImpl { +public: + AMDGPUPrivateObjectVGPRsImpl(const MachineFunction &MF, LiveIntervals *LIS) + : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()), LIS(LIS) {} + + bool run(MachineFunction &MF); + +private: + struct AllocaBBInfo { + const AllocaInst *Alloca = nullptr; + bool LiveIn = false; + bool Starts = false; + bool Ends = false; + }; + + ObjectRegs computeObjectRegs(const AllocaInst &Alloca) const; + + const SIInstrInfo *TII; + LiveIntervals *LIS; + + DenseMap<const AllocaInst *, std::pair<ObjectRegs, MachineMemOperand *>> + AllocaObjectRegs; +}; + +} // end anonymous namespace + +// The registers an object occupies: one per dword, starting at the register its +// byte address in the address space names. +ObjectRegs AMDGPUPrivateObjectVGPRsImpl::computeObjectRegs( + const AllocaInst &Alloca) const { + const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(Alloca); + unsigned Offset = MD.getAddress(); + assert(Offset % 4 == 0 && "Object is not dword aligned"); + + // An object that does not fill its last register still owns the whole of it. + unsigned BaseRegIdx = Offset / 4; + unsigned NumRegs = divideCeil(MD.getSize(), 4); + + ObjectRegs Regs; + Regs.reserve(NumRegs); + for (unsigned I : seq(NumRegs)) + Regs.push_back(AMDGPU::VGPR_32RegClass.getRegister(BaseRegIdx + I)); + + return Regs; +} + +bool AMDGPUPrivateObjectVGPRsImpl::run(MachineFunction &MF) { + // Sort basic blocks in reverse post-order for the live-out/live-in + // propagation. + DenseMap<MachineBasicBlock *, unsigned> BlockToIndex; + SmallVector<MachineBasicBlock *> IndexToBlock; + ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin()); + for (MachineBasicBlock *MBB : RPOT) { + BlockToIndex[MBB] = IndexToBlock.size(); + IndexToBlock.push_back(MBB); + } + + // Fixed-point iteration to determine basic block live-ins. + // + // The first pass of the fixed-point iteration also scans instructions. + SmallVector<SmallVector<AllocaBBInfo>> BBInfos(IndexToBlock.size()); + SmallBitVector Worklist(IndexToBlock.size()); + bool Changed = false; + + for (bool Dirty = true, FirstPass = true; Dirty; FirstPass = false) { + Dirty = false; + + for (auto [MBBI, MBB] : enumerate(IndexToBlock)) { + auto &BBI = BBInfos[MBBI]; + + // During the first outer iteration, augment VGPR_LIFETIME_{START,END} + // with implicit operands and record the initial per-basic block + // information to compute live-ins. + if (FirstPass) { + for (MachineInstr &MI : *MBB) { + if (MI.getOpcode() != AMDGPU::VGPR_LIFETIME_START && + MI.getOpcode() != AMDGPU::VGPR_LIFETIME_END) + continue; + + bool IsStart = MI.getOpcode() == AMDGPU::VGPR_LIFETIME_START; + MachineMemOperand *MMO = *MI.memoperands_begin(); + const auto *Alloca = cast<AllocaInst>(MMO->getValue()); + + auto ObjRegsIt = AllocaObjectRegs.find(Alloca); + if (ObjRegsIt == AllocaObjectRegs.end()) + ObjRegsIt = + AllocaObjectRegs + .try_emplace(Alloca, computeObjectRegs(*Alloca), MMO) + .first; + + // The object comes into existence at the start and dies at the end. + for (MCPhysReg Reg : ObjRegsIt->second.first) + MI.addOperand(MachineOperand::CreateReg( + Reg, /*isDef=*/IsStart, /*isImp=*/true, /*isKill=*/!IsStart)); + + auto It = find_if(BBI, [&](const AllocaBBInfo &Info) { + return Info.Alloca == Alloca; + }); + if (It == BBI.end()) { + BBI.push_back({Alloca, false, false, false}); + It = std::prev(BBI.end()); + } + It->Starts = IsStart; + It->Ends = !IsStart; + + Changed = true; + } + } else { + if (!Worklist[MBBI]) + continue; + Worklist[MBBI] = false; + } + + // Propagate live-outs into successors. + for (const auto &ABBI : BBI) { + if (!((ABBI.LiveIn && !ABBI.Ends) || ABBI.Starts)) + continue; + + const ObjectRegs &Regs = AllocaObjectRegs.at(ABBI.Alloca).first; + for (MachineBasicBlock *Succ : MBB->successors()) { + unsigned SuccI = BlockToIndex.at(Succ); + auto &SuccBBI = BBInfos[SuccI]; + auto It = find_if(SuccBBI, [&](const AllocaBBInfo &Info) { + return Info.Alloca == ABBI.Alloca; + }); + + bool Update = false; + if (It == SuccBBI.end()) { + SuccBBI.push_back({ABBI.Alloca, true, false, false}); + It = std::prev(SuccBBI.end()); + Update = true; + } else if (!It->LiveIn) { + It->LiveIn = true; + Update = true; + } + + if (!Update) + continue; + + // We are live-out from the successor because of the newly found + // live-in. If the successor is earlier in RPOT, we will have to + // re-evaluate it on the next outer iteration. + if (!It->Starts && !It->Ends && SuccI < MBBI) { + Worklist[SuccI] = true; + Dirty = true; + } + + for (MCPhysReg Reg : Regs) + Succ->addLiveIn(Reg); + } + } + } + } + + // It is legal for the pre-isel LLVM IR to have a lifetime.start without a + // lifetime.end. Liveness analysis is strong enough to mark physical registers + // as unused immediately after VGPR_LIFETIME_START in this case. + // + // Add VGPR_LIFETIME_END instructions at the end of basic blocks that end the + // function. + for (auto [BBIdx, MBB] : enumerate(IndexToBlock)) { + if (!MBB->succ_empty()) + continue; + + for (const auto &ABBI : BBInfos[BBIdx]) { + if (ABBI.Ends || (!ABBI.LiveIn && !ABBI.Starts)) + continue; + + // There may be a COPY to a conflicting physical VGPR before a function + // return, so put the end as late as possible: walk back over the + // instructions that cannot be observing the object. Anything that touches + // memory, calls, or has side effects ends the walk, since the object has + // to stay reserved across it. + MachineBasicBlock::iterator IP = MBB->getFirstTerminator(); + while (IP != MBB->begin()) { + --IP; + if (IP->mayStore() || IP->mayLoad() || IP->isCall() || + IP->hasUnmodeledSideEffects()) { + ++IP; + break; + } + } + + const auto &[ObjRegs, MMO] = AllocaObjectRegs.at(ABBI.Alloca); + MachineInstr *MI = + BuildMI(*MBB, IP, {}, TII->get(AMDGPU::VGPR_LIFETIME_END)) + .addMemOperand(MMO); + for (MCPhysReg Reg : ObjRegs) + MI->addOperand(MachineOperand::CreateReg( + Reg, /*isDef=*/false, /*isImp=*/true, /*isKill=*/true)); + + Changed = true; + } + } + + // An object lives in caller-saved registers, so a callee is free to overwrite + // it. Being live across a call is diagnosed rather than left to read back + // whatever the callee happened to leave behind. One diagnostic per object: + // the rest of its calls would say the same thing. + SmallPtrSet<const AllocaInst *, 4> Diagnosed; + for (auto [BBIdx, MBB] : enumerate(IndexToBlock)) { + for (const auto &ABBI : BBInfos[BBIdx]) { + if (Diagnosed.contains(ABBI.Alloca)) + continue; + + bool Live = ABBI.LiveIn; + for (const MachineInstr &MI : *MBB) { + if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(&MI)) { + if (&Marker->getObject() == ABBI.Alloca) + Live = Marker->isStart(); + continue; + } + if (!Live || !MI.isCall()) + continue; + + const Function &F = MF.getFunction(); + F.getContext().diagnose(DiagnosticInfoUnsupported( + F, + "object in the VGPR 'as memory' address space (13) is live across " + "a call", + MI.getDebugLoc())); + Diagnosed.insert(ABBI.Alloca); + break; + } + } + } + + // Remove live ranges from LiveIntervals. They will be recalculated lazily. + if (LIS) { + for (const auto &[Alloca, RegsAndMMO] : AllocaObjectRegs) { + for (MCPhysReg Reg : RegsAndMMO.first) + LIS->removeAllRegUnitsForPhysReg(Reg); + } + } + + return Changed; +} + +namespace { + +class AMDGPUPrivateObjectVGPRsLegacy : public MachineFunctionPass { +public: + static char ID; + + AMDGPUPrivateObjectVGPRsLegacy() : MachineFunctionPass(ID) {} + + bool runOnMachineFunction(MachineFunction &MF) override { + LiveIntervals *LIS = nullptr; + if (auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>()) + LIS = &LISWrapper->getLIS(); + return AMDGPUPrivateObjectVGPRsImpl(MF, LIS).run(MF); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + // LiveVariables only tracks virtual registers and we only touch physical + // registers. + AU.addPreserved<LiveVariablesWrapperPass>(); + AU.addPreserved<SlotIndexesWrapperPass>(); + AU.addPreserved<LiveIntervalsWrapperPass>(); + AU.addPreservedID(MachineLoopInfoID); + AU.addPreservedID(MachineDominatorsID); + MachineFunctionPass::getAnalysisUsage(AU); + } + + StringRef getPassName() const override { + return "AMDGPU Def/use private object VGPRs"; + } +}; + +} // end anonymous namespace + +PreservedAnalyses +AMDGPUPrivateObjectVGPRsPass::run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF); + if (!AMDGPUPrivateObjectVGPRsImpl(MF, LIS).run(MF)) + return PreservedAnalyses::all(); + + auto PA = getMachineFunctionPassPreservedAnalyses(); + PA.preserveSet<CFGAnalyses>(); + PA.preserve<LiveVariablesAnalysis>(); + PA.preserve<SlotIndexesAnalysis>(); + PA.preserve<LiveIntervalsAnalysis>(); + return PA; +} + +char AMDGPUPrivateObjectVGPRsLegacy::ID = 0; + +char &llvm::AMDGPUPrivateObjectVGPRsID = AMDGPUPrivateObjectVGPRsLegacy::ID; + +INITIALIZE_PASS(AMDGPUPrivateObjectVGPRsLegacy, DEBUG_TYPE, + "AMDGPU Def/use private object VGPRs", false, false) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp index 2223b9d036fa1..dc64a33187296 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp @@ -26,6 +26,7 @@ //===----------------------------------------------------------------------===// #include "AMDGPU.h" +#include "AMDGPUMemoryUtils.h" #include "GCNSubtarget.h" #include "Utils/AMDGPUBaseInfo.h" #include "llvm/ADT/STLExtras.h" @@ -142,6 +143,9 @@ class AMDGPUPromoteAllocaImpl { bool IsAMDGCN = false; bool IsAMDHSA = false; + /// Next free byte offset in the VGPR ("as memory") address space. + unsigned AllocVGPROffset = 0; + std::pair<Value *, Value *> getLocalSizeYZ(IRBuilder<> &Builder); Value *getWorkitemID(IRBuilder<> &Builder, unsigned N); @@ -167,6 +171,8 @@ class AMDGPUPromoteAllocaImpl { void finishDeferredAllocaToLDSPromotion(SetVector<IntrinsicInst *> &DeferredIntrs); + void allocateVgprs(AllocaAnalysis &AA); + void scoreAlloca(AllocaAnalysis &AA) const; void setFunctionLimits(const Function &F); @@ -179,7 +185,11 @@ class AMDGPUPromoteAllocaImpl { IsAMDHSA = TT.getOS() == Triple::AMDHSA; } - bool run(Function &F, bool PromoteToLDS); + /// IsLatePass is true if invoked as a codegen pass and false if invoked from + /// the optimization pipeline ("amdgpu-promote-alloca-to-vector" pass). NoOpt + /// indicates to perform only the codegen work that is strictly required for + /// functionality, which is allocating objects in the VGPR address space. + bool run(Function &F, bool IsLatePass, bool NoOpt = false); }; // FIXME: This can create globals so should be a module pass. @@ -187,7 +197,8 @@ class AMDGPUPromoteAlloca : public FunctionPass { public: static char ID; - AMDGPUPromoteAlloca() : FunctionPass(ID) {} + explicit AMDGPUPromoteAlloca(CodeGenOptLevel OptLevel) + : FunctionPass(ID), NoOpt(OptLevel == CodeGenOptLevel::None) {} bool runOnFunction(Function &F) override { if (skipFunction(F)) @@ -196,17 +207,22 @@ class AMDGPUPromoteAlloca : public FunctionPass { return AMDGPUPromoteAllocaImpl( TPC->getTM<TargetMachine>(), *F.getParent(), getAnalysis<LoopInfoWrapperPass>().getLoopInfo()) - .run(F, /*PromoteToLDS*/ true); + .run(F, /*IsLatePass=*/true, NoOpt); return false; } - StringRef getPassName() const override { return "AMDGPU Promote Alloca"; } + StringRef getPassName() const override { + return NoOpt ? "AMDGPU VGPR Allocate" : "AMDGPU Promote Alloca"; + } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); AU.addRequired<LoopInfoWrapperPass>(); FunctionPass::getAnalysisUsage(AU); } + +private: + bool NoOpt; }; static unsigned getMaxVGPRs(unsigned LDSBytes, const TargetMachine &TM, @@ -251,7 +267,7 @@ PreservedAnalyses AMDGPUPromoteAllocaPass::run(Function &F, FunctionAnalysisManager &AM) { auto &LI = AM.getResult<LoopAnalysis>(F); bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI) - .run(F, /*PromoteToLDS=*/true); + .run(F, /*IsLatePass=*/true); if (Changed) { PreservedAnalyses PA; PA.preserveSet<CFGAnalyses>(); @@ -264,7 +280,7 @@ PreservedAnalyses AMDGPUPromoteAllocaToVectorPass::run(Function &F, FunctionAnalysisManager &AM) { auto &LI = AM.getResult<LoopAnalysis>(F); bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI) - .run(F, /*PromoteToLDS=*/false); + .run(F, /*IsLatePass=*/false); if (Changed) { PreservedAnalyses PA; PA.preserveSet<CFGAnalyses>(); @@ -273,8 +289,21 @@ AMDGPUPromoteAllocaToVectorPass::run(Function &F, FunctionAnalysisManager &AM) { return PreservedAnalyses::all(); } -FunctionPass *llvm::createAMDGPUPromoteAlloca() { - return new AMDGPUPromoteAlloca(); +FunctionPass *llvm::createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel) { + return new AMDGPUPromoteAlloca(OptLevel); +} + +PreservedAnalyses AMDGPUVGPRAllocatePass::run(Function &F, + FunctionAnalysisManager &AM) { + auto &LI = AM.getResult<LoopAnalysis>(F); + bool Changed = AMDGPUPromoteAllocaImpl(TM, *F.getParent(), LI) + .run(F, /*IsLatePass=*/true, /*NoOpt=*/true); + if (!Changed) + return PreservedAnalyses::all(); + + PreservedAnalyses PA; + PA.preserveSet<CFGAnalyses>(); + return PA; } bool AMDGPUPromoteAllocaImpl::collectAllocaUses(AllocaAnalysis &AA) const { @@ -367,10 +396,16 @@ void AMDGPUPromoteAllocaImpl::setFunctionLimits(const Function &F) { VGPRBudgetRatio = PromoteAllocaToVectorVGPRRatio; } -bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) { - if (DisablePromoteAllocaToLDS && DisablePromoteAllocaToVector) +bool AMDGPUPromoteAllocaImpl::run(Function &F, bool IsLatePass, bool NoOpt) { + assert((!NoOpt || IsLatePass) && "NoOpt only makes sense for the late pass"); + + // Without optimizations only the allocation below is done, so the promotion + // options do not get a say. + if (!NoOpt && DisablePromoteAllocaToLDS && DisablePromoteAllocaToVector) return false; + const bool PromoteToLDS = IsLatePass && !NoOpt; + bool SufficientLDS = PromoteToLDS && hasSufficientLocalMem(F); MaxVGPRs = IsAMDGCN ? getMaxVGPRs(CurrentLocalMemUsage, TM, F) : 128; setFunctionLimits(F); @@ -391,6 +426,21 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) { LLVM_DEBUG(dbgs() << "Analyzing: " << *AI << '\n'); AllocaAnalysis AA{AI}; + + // An alloca that is already in the VGPR ("as memory") address space is + // not promoted, only given a place in that address space, which only the + // codegen pass does. It happens whether or not optimizations are enabled, + // since the address space cannot be used without it. + if (AI->getAddressSpace() == AMDGPUAS::VGPR) { + if (IsLatePass) + Allocas.push_back(std::move(AA)); + continue; + } + + // Everything below this point is an optimization. + if (NoOpt) + continue; + if (collectAllocaUses(AA)) { analyzePromoteToVector(AA); if (PromoteToLDS) @@ -403,8 +453,15 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) { } } - stable_sort(Allocas, - [](const auto &A, const auto &B) { return A.Score > B.Score; }); + stable_sort(Allocas, [](const auto &A, const auto &B) { + // Prioritize allocas that are already in the VGPR address space, since + // their allocation must not fail. + bool AIsVGPR = A.Alloca->getAddressSpace() == AMDGPUAS::VGPR; + bool BIsVGPR = B.Alloca->getAddressSpace() == AMDGPUAS::VGPR; + if (AIsVGPR != BIsVGPR) + return AIsVGPR; + return A.Score > B.Score; + }); // clang-format off LLVM_DEBUG( @@ -417,6 +474,17 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) { bool Changed = false; SetVector<IntrinsicInst *> DeferredIntrs; for (AllocaAnalysis &AA : Allocas) { + if (AA.Alloca->getAddressSpace() == AMDGPUAS::VGPR) { + allocateVgprs(AA); + std::optional<TypeSize> Size = AA.Alloca->getAllocationSize(DL); + assert(Size); // Expected to succeed on non-array alloca. + const unsigned AllocaCost = Size->getFixedValue() * 8; + // These registers are no longer available for vectorization. + VectorizationBudget -= std::min(VectorizationBudget, AllocaCost); + Changed = true; + continue; + } + if (AA.Vector.Ty) { std::optional<TypeSize> Size = AA.Alloca->getAllocationSize(DL); assert(Size); // Expected to succeed on non-array alloca. @@ -451,6 +519,66 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) { return Changed; } +// Give an alloca in the VGPR ("as memory") address space a place in that +// address space, recorded on it as !amdgpu.allocated.vgprs, and lifetime +// markers the backend understands. AMDGPUPrivateObjectVGPRs turns the two into +// defs and uses of the physical registers the object occupies, which is what +// keeps register allocation off them while it is live. +void AMDGPUPromoteAllocaImpl::allocateVgprs(AllocaAnalysis &AA) { + LLVMContext &Ctx = Mod.getContext(); + const unsigned AllocaSize = + DL.getTypeAllocSize(AA.Alloca->getAllocatedType()).getFixedValue(); + + // The generic lifetime intrinsics do not survive into the backend, so use + // the address-space specific ones instead. An object with no lifetime start + // is live from its definition. + bool HaveLifetimeStart = false; + for (Use &U : AA.Alloca->uses()) { + auto *II = dyn_cast<IntrinsicInst>(U.getUser()); + if (!II) + continue; + if (II->getIntrinsicID() == Intrinsic::lifetime_start) { + HaveLifetimeStart = true; + II->setCalledFunction(Intrinsic::getOrInsertDeclaration( + &Mod, Intrinsic::amdgcn_vgpr_lifetime_start, AA.Alloca->getType())); + } else if (II->getIntrinsicID() == Intrinsic::lifetime_end) { + II->setCalledFunction(Intrinsic::getOrInsertDeclaration( + &Mod, Intrinsic::amdgcn_vgpr_lifetime_end, AA.Alloca->getType())); + } + } + + if (!HaveLifetimeStart) { + Instruction *IP = AA.Alloca->getNextNode(); + while (isa<AllocaInst>(IP)) + IP = IP->getNextNode(); + IRBuilder<> B(IP); + B.SetCurrentDebugLocation(AA.Alloca->getDebugLoc()); + B.CreateIntrinsic(B.getVoidTy(), Intrinsic::amdgcn_vgpr_lifetime_start, + AA.Alloca); + } + + // Allocating twice would place a second object on top of this one, so an + // object that already has its place keeps it, and the next one is allocated + // after it. The markers above are still rewritten, since IR that carries the + // metadata need not carry them. + if (MDNode *MD = AA.Alloca->getMetadata("amdgpu.allocated.vgprs")) { + const auto &Allocated = *cast<AMDGPU::AllocatedVGPRsMetadata>(MD); + AllocVGPROffset = std::max(AllocVGPROffset, + Allocated.getAddress() + alignTo(AllocaSize, 4)); + return; + } + + // Find space for the alloca. Objects are packed in order at dword + // granularity, since a register cannot be shared between two of them. + Type *I32 = Type::getInt32Ty(Ctx); + AA.Alloca->setMetadata( + "amdgpu.allocated.vgprs", + MDNode::get( + Ctx, {ConstantAsMetadata::get(ConstantInt::get(I32, AllocVGPROffset)), + ConstantAsMetadata::get(ConstantInt::get(I32, AllocaSize))})); + AllocVGPROffset += alignTo(AllocaSize, 4); +} + // Checks if the instruction I is a memset user of the alloca AI that we can // deal with. Currently, only non-volatile memsets that affect the whole alloca // are handled. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index f296570ecce8c..7831054335b3a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -669,6 +669,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { initializeSILowerWWMCopiesLegacyPass(*PR); initializeAMDGPUMarkLastScratchLoadLegacyPass(*PR); initializeAMDGPULowerIdxOpsLegacyPass(*PR); + initializeAMDGPUPrivateObjectVGPRsLegacyPass(*PR); initializeSILowerSGPRSpillsLegacyPass(*PR); initializeSIFixSGPRCopiesLegacyPass(*PR); initializeSIFixVGPRCopiesLegacyPass(*PR); @@ -1557,9 +1558,12 @@ void AMDGPUPassConfig::addIRPasses() { addPass(createAtomicExpandLegacyPass()); - if (TM.getOptLevel() > CodeGenOptLevel::None) { - addPass(createAMDGPUPromoteAlloca()); + // With optimizations enabled, do the full promotion of allocas. Without + // optimizations, only allocate existing addrspace(13) allocas, which is + // needed for functionality rather than for performance. + addPass(createAMDGPUPromoteAlloca(TM.getOptLevel())); + if (TM.getOptLevel() > CodeGenOptLevel::None) { if (isPassEnabled(EnableScalarIRPasses)) addStraightLineScalarOptimizationPasses(); @@ -1774,6 +1778,8 @@ void GCNPassConfig::addFastRegAlloc() { // SI_ELSE will introduce a copy of the tied operand source after the else. insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID); + insertPass(&PHIEliminationID, &AMDGPUPrivateObjectVGPRsID); + insertPass(&TwoAddressInstructionPassID, &SIWholeQuadModeID); TargetPassConfig::addFastRegAlloc(); @@ -1806,6 +1812,8 @@ void GCNPassConfig::addOptimizedRegAlloc() { // SI_ELSE will introduce a copy of the tied operand source after the else. insertPass(&PHIEliminationID, &SILowerControlFlowLegacyID); + insertPass(&PHIEliminationID, &AMDGPUPrivateObjectVGPRsID); + if (EnableRewritePartialRegUses) insertPass(&RenameIndependentSubregsID, &GCNRewritePartialRegUsesID); @@ -2346,8 +2354,13 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const { addFunctionPass(AtomicExpandPass(TM), PMW); - if (TM.getOptLevel() > CodeGenOptLevel::None) { + // See the comment on the legacy pipeline above. + if (TM.getOptLevel() > CodeGenOptLevel::None) addFunctionPass(AMDGPUPromoteAllocaPass(TM), PMW); + else + addFunctionPass(AMDGPUVGPRAllocatePass(TM), PMW); + + if (TM.getOptLevel() > CodeGenOptLevel::None) { if (isPassEnabled(EnableScalarIRPasses)) addStraightLineScalarOptimizationPasses(PMW); @@ -2518,6 +2531,8 @@ void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization( Error AMDGPUCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) const { insertPass<PHIEliminationPass>(SILowerControlFlowPass()); + insertPass<PHIEliminationPass>(AMDGPUPrivateObjectVGPRsPass()); + insertPass<TwoAddressInstructionPass>(SIWholeQuadModePass()); return Base::addFastRegAlloc(PMW); @@ -2580,6 +2595,8 @@ Error AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc( // SI_ELSE will introduce a copy of the tied operand source after the else. insertPass<PHIEliminationPass>(SILowerControlFlowPass()); + insertPass<PHIEliminationPass>(AMDGPUPrivateObjectVGPRsPass()); + if (EnableRewritePartialRegUses) insertPass<RenameIndependentSubregsPass>(GCNRewritePartialRegUsesPass()); diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt index 5c9f8a2063847..6853b3f47081c 100644 --- a/llvm/lib/Target/AMDGPU/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt @@ -92,6 +92,7 @@ add_llvm_target(AMDGPUCodeGen AMDGPUIGroupLP.cpp AMDGPULowerIdxOps.cpp AMDGPULowerVGPREncoding.cpp + AMDGPUPrivateObjectVGPRs.cpp AMDGPUMCResourceInfo.cpp AMDGPUMarkLastScratchLoad.cpp AMDGPUMIRFormatter.cpp diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 9f334be564f7f..5376fc8aa5429 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -278,6 +278,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand); setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom); + + // Only to give an alloca in the VGPR ("as memory") address space the address + // it was allocated; every other frame index is left alone. + setOperationAction(ISD::FrameIndex, MVT::i32, Custom); setOperationAction(ISD::BlockAddress, {MVT::i32, MVT::i64}, Custom); setOperationAction(ISD::ExternalSymbol, {MVT::i32, MVT::i64}, Custom); @@ -1574,6 +1578,22 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos, Infos.push_back(Info); return; } + case Intrinsic::amdgcn_vgpr_lifetime_start: + case Intrinsic::amdgcn_vgpr_lifetime_end: { + // A marker covers the whole object, whose size and placement within the + // VGPR ("as memory") address space AMDGPUPromoteAlloca recorded on the + // alloca. Carrying that as a memory operand is what lets + // AMDGPUPrivateObjectVGPRs find the registers the object occupies. + const auto *Alloca = cast<AllocaInst>(CI.getArgOperand(0)); + const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca); + Info.opc = ISD::INTRINSIC_VOID; + Info.memVT = EVT::getIntegerVT(CI.getContext(), MD.getSize() * 8); + Info.ptrVal = Alloca; + Info.flags = Flags | MachineMemOperand::MODereferenceable | + MachineMemOperand::MOStore; + Infos.push_back(Info); + return; + } case Intrinsic::amdgcn_ds_append: case Intrinsic::amdgcn_ds_consume: { Info.opc = ISD::INTRINSIC_W_CHAIN; @@ -7575,6 +7595,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return LowerBRCOND(Op, DAG); case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); + case ISD::FrameIndex: + return lowerFrameIndex(Op, DAG); case ISD::SPONENTRY: return LowerSPONENTRY(Op, DAG); case ISD::LOAD: { @@ -10086,6 +10108,20 @@ SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI, MachineMemOperand::MOInvariant); } +// The address of an object in the VGPR ("as memory") address space is where +// AMDGPUPromoteAlloca placed it, recorded on the alloca. Every other frame +// index keeps its default lowering. +SDValue SITargetLowering::lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const { + MachineFunction &MF = DAG.getMachineFunction(); + int FI = cast<FrameIndexSDNode>(Op)->getIndex(); + const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI); + if (!Alloca || Alloca->getAddressSpace() != AMDGPUAS::VGPR) + return SDValue(); + + const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca); + return DAG.getConstant(MD.getAddress(), SDLoc(Op), MVT::i32); +} + SDValue SITargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const { // TODO: Handle this. It should be mostly the same as LowerGlobalAddress. @@ -12295,6 +12331,15 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, unsigned IntrinsicID = Op.getConstantOperandVal(1); switch (IntrinsicID) { + case Intrinsic::amdgcn_vgpr_lifetime_start: + case Intrinsic::amdgcn_vgpr_lifetime_end: { + unsigned Opcode = IntrinsicID == Intrinsic::amdgcn_vgpr_lifetime_start + ? AMDGPU::VGPR_LIFETIME_START + : AMDGPU::VGPR_LIFETIME_END; + MachineSDNode *Marker = DAG.getMachineNode(Opcode, DL, MVT::Other, Chain); + DAG.setNodeMemRefs(Marker, {cast<MemSDNode>(Op)->getMemOperand()}); + return SDValue(Marker, 0); + } case Intrinsic::amdgcn_exp_compr: { if (!Subtarget->hasCompressedExport()) { DAG.getContext()->diagnose(DiagnosticInfoUnsupported( @@ -19990,6 +20035,17 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const { Info->limitOccupancy(MF); + // An object in the VGPR "as memory" address space lives in registers, and + // every reference to its frame index has been replaced by the address it was + // allocated, so it must not take up stack space as well. + MachineFrameInfo &MFI = MF.getFrameInfo(); + for (int FI = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); FI != E; + ++FI) { + const AllocaInst *Alloca = MFI.getObjectAllocation(FI); + if (Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR) + MFI.RemoveStackObject(FI); + } + // Give a VGPR "as memory" indexed access its M0 operand, matching how // AMDGPULowerVGPREncoding will expand it. Under the VGPR indexing mode the // expansion emits an s_set_gpr_idx_on, which reads the index out of its SGPR diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index f270a03422757..1f0008b392b3a 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -182,6 +182,7 @@ class SITargetLowering final : public AMDGPUTargetLowering { SDValue getSegmentAperture(unsigned AS, const SDLoc &DL, SelectionDAG &DAG) const; + SDValue lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const; SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const; SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index d5d1e1f44fb99..c5ddc84ba13d7 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4064,13 +4064,16 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, return false; // VGPR "as memory" indexed accesses only alias each other, and then only - // when their [idx+offset, idx+offset+width) dword ranges overlap. + // when their [idx+offset, idx+offset+width) dword ranges overlap. The + // exception is a lifetime marker, which conceptually overwrites the whole + // object it covers and so may alias any access to it. const bool IsLdStIdxA = isa<AMDGPUMI::VLoadStoreIdxInst>(MIa); const bool IsLdStIdxB = isa<AMDGPUMI::VLoadStoreIdxInst>(MIb); if (IsLdStIdxA || IsLdStIdxB) { if (IsLdStIdxA && IsLdStIdxB) return checkInstOffsetsDoNotOverlap(MIa, MIb); - return true; + return !isa<AMDGPUMI::VGPRLifetimeInst>(MIa) && + !isa<AMDGPUMI::VGPRLifetimeInst>(MIb); } if (isLDSDMA(MIa) || isLDSDMA(MIb)) diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 1b66711c4fbe3..c836af2f36dd3 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -1161,6 +1161,25 @@ foreach vt = !listconcat( Reg512Types.types, Reg1024Types.types) in defm : VRegIdxLoadStorePat<vt>; +/// Mark the start and end of the lifetime of an object in the VGPR ("as +/// memory") address space. AMDGPUPrivateObjectVGPRs gives these the physical +/// VGPRs the object was allocated to, as an implicit def on the start and an +/// implicit killed use on the end, which is what keeps register allocation off +/// those registers while the object is live. +/// +/// These instructions conceptually overwrite the referenced memory with +/// "undef". +def VGPR_LIFETIME_START : SPseudoInstSI<(outs), (ins)> { + let hasSideEffects = false; + let mayStore = true; + let isMeta = true; +} +def VGPR_LIFETIME_END : SPseudoInstSI<(outs), (ins)> { + let hasSideEffects = false; + let mayStore = true; + let isMeta = true; +} + // Used for byte or short loads. // It lowers into V_BFE_U32 + V_LOAD_IDX, or V_BFE_I32 + V_LOAD_IDX for // sign-extended loads. diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll new file mode 100644 index 0000000000000..d18ed7488ab69 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll @@ -0,0 +1,70 @@ +; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s +; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00-- -filetype=null %s 2>&1 | FileCheck %s + +; An object in the VGPR "as memory" address space (13) is allocated to +; caller-saved registers, so a callee is free to overwrite it. Being live across +; a call is therefore diagnosed rather than quietly reading back whatever the +; callee left behind. + +declare void @extern_func() +declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture) +declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture) + +; CHECK: error: {{.*}}in function across_call{{.*}}object in the VGPR 'as memory' address space (13) is live across a call +define void @across_call(ptr addrspace(1) %out, i32 %i) { + %obj = alloca [4 x i32], addrspace(13) + %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i + store i32 7, ptr addrspace(13) %p + call void @extern_func() + %v = load volatile i32, ptr addrspace(13) %p + store i32 %v, ptr addrspace(1) %out + ret void +} + +; The lifetime ends before the call, so the registers are free by then. +; CHECK-NOT: in function dies_before_call +define void @dies_before_call(ptr addrspace(1) %out, i32 %i) { + %obj = alloca [4 x i32], addrspace(13) + call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj) + %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i + store i32 7, ptr addrspace(13) %p + %v = load volatile i32, ptr addrspace(13) %p + call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj) + call void @extern_func() + store i32 %v, ptr addrspace(1) %out + ret void +} + +; Reachability, not textual order: the object is still live on the path through +; the call. +; CHECK: error: {{.*}}in function across_call_in_branch{{.*}}live across a call +define void @across_call_in_branch(ptr addrspace(1) %out, i32 %i, i1 %c) { +entry: + %obj = alloca [4 x i32], addrspace(13) + call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj) + %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i + store i32 7, ptr addrspace(13) %p + br i1 %c, label %call, label %join + +call: + call void @extern_func() + br label %join + +join: + %v = load volatile i32, ptr addrspace(13) %p + call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj) + store i32 %v, ptr addrspace(1) %out + ret void +} + +; An object with no lifetime end is live to the end of the function, so a call +; in the exit block is across it too: the synthesized end goes after the call, +; not before it. +; CHECK: error: {{.*}}in function call_in_exit_block{{.*}}live across a call +define void @call_in_exit_block(ptr addrspace(1) %out, i32 %i) { + %obj = alloca [4 x i32], addrspace(13) + %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i + store i32 7, ptr addrspace(13) %p + call void @extern_func() + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll new file mode 100644 index 0000000000000..652798f0278a8 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll @@ -0,0 +1,133 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-promote-alloca -o - %s | FileCheck %s + +; An alloca in the VGPR "as memory" address space (13) is not promoted but +; allocated: AMDGPUPromoteAlloca gives it a place in that address space, +; recorded as !amdgpu.allocated.vgprs {byte offset, byte size}, and lifetime +; markers the backend turns into defs and uses of the registers it occupies. + +define amdgpu_kernel void @one_object(ptr addrspace(1) %out, i32 %i) { +; CHECK-LABEL: define amdgpu_kernel void @one_object( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[I:%.*]]) { +; CHECK-NEXT: [[OBJ:%.*]] = alloca [4 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META0:![0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]]) +; CHECK-NEXT: [[P:%.*]] = getelementptr [4 x i32], ptr addrspace(13) [[OBJ]], i32 0, i32 [[I]] +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[P]], align 4 +; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4 +; CHECK-NEXT: ret void +; + %obj = alloca [4 x i32], addrspace(13) + %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i + %v = load i32, ptr addrspace(13) %p + store i32 %v, ptr addrspace(1) %out + ret void +} + +; Objects are packed in order, at dword granularity, since a register cannot be +; shared between two of them. +define amdgpu_kernel void @packed_objects(ptr addrspace(1) %out) { +; CHECK-LABEL: define amdgpu_kernel void @packed_objects( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) { +; CHECK-NEXT: [[A:%.*]] = alloca i8, align 1, addrspace(13), !amdgpu.allocated.vgprs [[META1:![0-9]+]] +; CHECK-NEXT: [[B:%.*]] = alloca [2 x i32], align 4, addrspace(13), !amdgpu.allocated.vgprs [[META2:![0-9]+]] +; CHECK-NEXT: [[C:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META3:![0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[C]]) +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[B]]) +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[A]]) +; CHECK-NEXT: [[VA:%.*]] = load i8, ptr addrspace(13) [[A]], align 1 +; CHECK-NEXT: [[VB:%.*]] = load i32, ptr addrspace(13) [[B]], align 4 +; CHECK-NEXT: [[VC:%.*]] = load i32, ptr addrspace(13) [[C]], align 4 +; CHECK-NEXT: [[EA:%.*]] = zext i8 [[VA]] to i32 +; CHECK-NEXT: [[S0:%.*]] = add i32 [[EA]], [[VB]] +; CHECK-NEXT: [[S1:%.*]] = add i32 [[S0]], [[VC]] +; CHECK-NEXT: store i32 [[S1]], ptr addrspace(1) [[OUT]], align 4 +; CHECK-NEXT: ret void +; + %a = alloca i8, addrspace(13) + %b = alloca [2 x i32], addrspace(13) + %c = alloca i32, addrspace(13) + %va = load i8, ptr addrspace(13) %a + %vb = load i32, ptr addrspace(13) %b + %vc = load i32, ptr addrspace(13) %c + %ea = zext i8 %va to i32 + %s0 = add i32 %ea, %vb + %s1 = add i32 %s0, %vc + store i32 %s1, ptr addrspace(1) %out + ret void +} + +; Existing lifetime markers become the address space specific ones rather than +; a synthesized start. +define amdgpu_kernel void @existing_lifetimes(ptr addrspace(1) %out) { +; CHECK-LABEL: define amdgpu_kernel void @existing_lifetimes( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) { +; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META4:![0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]]) +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4 +; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4 +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]]) +; CHECK-NEXT: ret void +; + %obj = alloca i32, addrspace(13) + call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj) + %v = load i32, ptr addrspace(13) %obj + store i32 %v, ptr addrspace(1) %out + call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj) + ret void +} + +; Running again must not allocate a second place on top of the first, and must +; not add a second marker. +define amdgpu_kernel void @already_allocated(ptr addrspace(1) %out) { +; CHECK-LABEL: define amdgpu_kernel void @already_allocated( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) { +; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META5:![0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]]) +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]]) +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4 +; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4 +; CHECK-NEXT: ret void +; + %obj = alloca i32, addrspace(13), !amdgpu.allocated.vgprs !0 + call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) %obj) + %v = load i32, ptr addrspace(13) %obj + store i32 %v, ptr addrspace(1) %out + ret void +} + +; Metadata without the address-space specific markers: the place is kept, but +; the generic markers still have to be rewritten, or nothing reserves the +; registers in the backend. +define amdgpu_kernel void @allocated_but_generic_markers(ptr addrspace(1) %out) { +; CHECK-LABEL: define amdgpu_kernel void @allocated_but_generic_markers( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) { +; CHECK-NEXT: [[OBJ:%.*]] = alloca i32, align 4, addrspace(13), !amdgpu.allocated.vgprs [[META6:![0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) [[OBJ]]) +; CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(13) [[OBJ]], align 4 +; CHECK-NEXT: store i32 [[V]], ptr addrspace(1) [[OUT]], align 4 +; CHECK-NEXT: call void @llvm.amdgcn.vgpr.lifetime.end.p13(ptr addrspace(13) [[OBJ]]) +; CHECK-NEXT: ret void +; + %obj = alloca i32, addrspace(13), !amdgpu.allocated.vgprs !1 + call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj) + %v = load i32, ptr addrspace(13) %obj + store i32 %v, ptr addrspace(1) %out + call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj) + ret void +} + +declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture) +declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture) +declare void @llvm.amdgcn.vgpr.lifetime.start.p13(ptr addrspace(13) nocapture) + +!0 = !{i32 8, i32 4} +!1 = !{i32 32, i32 4} +;. +; CHECK: [[META0]] = !{i32 0, i32 16} +; CHECK: [[META1]] = !{i32 0, i32 1} +; CHECK: [[META2]] = !{i32 4, i32 8} +; CHECK: [[META3]] = !{i32 12, i32 4} +; CHECK: [[META4]] = !{i32 0, i32 4} +; CHECK: [[META5]] = !{i32 8, i32 4} +; CHECK: [[META6]] = !{i32 32, i32 4} +;. diff --git a/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll new file mode 100644 index 0000000000000..f1b9b8d378cdb --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll @@ -0,0 +1,178 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG +; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefixes=GFX12,GFX12-GISEL +; Allocating the object is needed for functionality, not performance, so it +; happens without optimizations too. +; RUN: llc -O0 -global-isel=0 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefix=NOOPT +; RUN: llc -O0 -global-isel=1 -mtriple=amdgpu12.00-- -o - %s | FileCheck %s --check-prefix=NOOPT + +; NOOPT: avoids_object_registers: +; NOOPT: ; VGPR lifetime start: v[0:1] +; NOOPT: v_movrel{{[sd]}}_b32 +; NOOPT: ; VGPR lifetime end: v[0:1] +; NOOPT: ScratchSize: 0 + +; The registers an object in the VGPR "as memory" address space (13) was +; allocated must not be handed to anything else while it is live. The lifetime +; markers carry them as an implicit def and an implicit killed use, so the +; register allocator sees the object the same way it sees any other value. + +; The object occupies v[0:1], so the values live across it are allocated +; elsewhere. +define amdgpu_kernel void @avoids_object_registers(ptr addrspace(1) %out, i32 %i) { +; GFX12-SDAG-LABEL: avoids_object_registers: +; GFX12-SDAG: ; %bb.0: +; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 +; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, 7 :: v_dual_mov_b32 v3, 0 +; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:1] +; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 +; GFX12-SDAG-NEXT: s_mov_b32 m0, s2 +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v2 +; GFX12-SDAG-NEXT: global_load_b32 v2, v3, s[0:1] scope:SCOPE_SYS +; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX12-SDAG-NEXT: global_load_b32 v4, v3, s[0:1] scope:SCOPE_SYS +; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX12-SDAG-NEXT: global_load_b32 v5, v3, s[0:1] scope:SCOPE_SYS +; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:1] +; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v4 +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, v2, v5 +; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v2, 7, v2 +; GFX12-SDAG-NEXT: global_store_b32 v3, v2, s[0:1] +; GFX12-SDAG-NEXT: s_endpgm +; +; GFX12-GISEL-LABEL: avoids_object_registers: +; GFX12-GISEL: ; %bb.0: +; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 +; GFX12-GISEL-NEXT: v_dual_mov_b32 v2, 7 :: v_dual_mov_b32 v3, 0 +; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:1] +; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-NEXT: s_lshl_b32 s2, s2, 2 +; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-NEXT: s_lshr_b32 m0, s2, 2 +; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v2 +; GFX12-GISEL-NEXT: global_load_b32 v2, v3, s[0:1] scope:SCOPE_SYS +; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX12-GISEL-NEXT: global_load_b32 v4, v3, s[0:1] scope:SCOPE_SYS +; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX12-GISEL-NEXT: global_load_b32 v5, v3, s[0:1] scope:SCOPE_SYS +; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2 +; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v4 +; GFX12-GISEL-NEXT: v_readfirstlane_b32 s4, v5 +; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0 +; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:1] +; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s3 +; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s4 +; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX12-GISEL-NEXT: v_add_nc_u32_e32 v2, s2, v2 +; GFX12-GISEL-NEXT: global_store_b32 v3, v2, s[0:1] +; GFX12-GISEL-NEXT: s_endpgm + %obj = alloca [2 x i32], addrspace(13) + %p = getelementptr [2 x i32], ptr addrspace(13) %obj, i32 0, i32 %i + store i32 7, ptr addrspace(13) %p + %a = load volatile i32, ptr addrspace(1) %out + %b = load volatile i32, ptr addrspace(1) %out + %c = load volatile i32, ptr addrspace(1) %out + %v = load i32, ptr addrspace(13) %p + %s0 = add i32 %a, %b + %s1 = add i32 %s0, %c + %s2 = add i32 %s1, %v + store i32 %s2, ptr addrspace(1) %out + ret void +} + +; The object lives in registers, so it takes no stack space. +define amdgpu_kernel void @no_scratch(ptr addrspace(1) %out, i32 %i) { +; GFX12-SDAG-LABEL: no_scratch: +; GFX12-SDAG: ; %bb.0: +; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 +; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 7 +; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:3] +; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 +; GFX12-SDAG-NEXT: s_mov_b32 m0, s2 +; GFX12-SDAG-NEXT: global_store_b32 v4, v5, s[0:1] +; GFX12-SDAG-NEXT: v_movreld_b32_e32 v0, v5 +; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:3] +; GFX12-SDAG-NEXT: s_endpgm +; +; GFX12-GISEL-LABEL: no_scratch: +; GFX12-GISEL: ; %bb.0: +; GFX12-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 +; GFX12-GISEL-NEXT: v_dual_mov_b32 v4, 7 :: v_dual_mov_b32 v5, 0 +; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:3] +; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-NEXT: s_bfe_u32 m0, s2, 0x1e0000 +; GFX12-GISEL-NEXT: global_store_b32 v5, v4, s[0:1] +; GFX12-GISEL-NEXT: v_movreld_b32_e32 v0, v4 +; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:3] +; GFX12-GISEL-NEXT: s_endpgm + %obj = alloca [4 x i32], addrspace(13) + %p = getelementptr [4 x i32], ptr addrspace(13) %obj, i32 0, i32 %i + store i32 7, ptr addrspace(13) %p + %v = load i32, ptr addrspace(13) %p + store i32 %v, ptr addrspace(1) %out + ret void +} + +; Registers are only unavailable where the object is live: after its end they +; can be used again. +define amdgpu_kernel void @registers_reused_after_end(ptr addrspace(1) %out) { +; GFX12-SDAG-LABEL: registers_reused_after_end: +; GFX12-SDAG: ; %bb.0: +; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 +; GFX12-SDAG-NEXT: s_mov_b32 m0, 0 +; GFX12-SDAG-NEXT: ; VGPR lifetime start: v[0:0] +; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX12-SDAG-NEXT: v_movrels_b32_e32 v2, v0 +; GFX12-SDAG-NEXT: ; VGPR lifetime end: v[0:0] +; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 +; GFX12-SDAG-NEXT: global_store_b32 v1, v2, s[0:1] +; GFX12-SDAG-NEXT: global_load_b32 v0, v1, s[0:1] scope:SCOPE_SYS +; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX12-SDAG-NEXT: global_load_b32 v2, v1, s[0:1] scope:SCOPE_SYS +; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX12-SDAG-NEXT: v_add_nc_u32_e32 v0, v0, v2 +; GFX12-SDAG-NEXT: global_store_b32 v1, v0, s[0:1] +; GFX12-SDAG-NEXT: s_endpgm +; +; GFX12-GISEL-LABEL: registers_reused_after_end: +; GFX12-GISEL: ; %bb.0: +; GFX12-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 +; GFX12-GISEL-NEXT: s_mov_b32 m0, 0 +; GFX12-GISEL-NEXT: ; VGPR lifetime start: v[0:0] +; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0 +; GFX12-GISEL-NEXT: v_movrels_b32_e32 v2, v0 +; GFX12-GISEL-NEXT: ; VGPR lifetime end: v[0:0] +; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-NEXT: global_store_b32 v1, v2, s[0:1] +; GFX12-GISEL-NEXT: global_load_b32 v0, v1, s[0:1] scope:SCOPE_SYS +; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX12-GISEL-NEXT: global_load_b32 v2, v1, s[0:1] scope:SCOPE_SYS +; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0 +; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v2 +; GFX12-GISEL-NEXT: s_add_co_i32 s2, s2, s3 +; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2 +; GFX12-GISEL-NEXT: global_store_b32 v1, v0, s[0:1] +; GFX12-GISEL-NEXT: s_endpgm + %obj = alloca i32, addrspace(13) + call void @llvm.lifetime.start.p13(ptr addrspace(13) %obj) + %v = load i32, ptr addrspace(13) %obj + store i32 %v, ptr addrspace(1) %out + call void @llvm.lifetime.end.p13(ptr addrspace(13) %obj) + %a = load volatile i32, ptr addrspace(1) %out + %b = load volatile i32, ptr addrspace(1) %out + %s = add i32 %a, %b + store i32 %s, ptr addrspace(1) %out + ret void +} + +declare void @llvm.lifetime.start.p13(ptr addrspace(13) nocapture) +declare void @llvm.lifetime.end.p13(ptr addrspace(13) nocapture) +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX12: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll index f465ee4c0470c..e66de5ade3a63 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll @@ -29,6 +29,7 @@ ; GCN-O0-NEXT: amdgpu-lower-module-lds ; GCN-O0-NEXT: function ; GCN-O0-NEXT: atomic-expand +; GCN-O0-NEXT: amdgpu-vgpr-allocate ; GCN-O0-NEXT: verify ; GCN-O0-NEXT: unreachableblockelim ; GCN-O0-NEXT: ee-instrument<post-inline> @@ -72,6 +73,7 @@ ; GCN-O0-NEXT: amdgpu-lower-idx-ops ; GCN-O0-NEXT: phi-node-elimination ; GCN-O0-NEXT: si-lower-control-flow +; GCN-O0-NEXT: amdgpu-private-object-vgprs ; GCN-O0-NEXT: two-address-instruction ; GCN-O0-NEXT: si-wqm ; GCN-O0-NEXT: amdgpu-pre-ra-long-branch-reg @@ -230,6 +232,7 @@ ; GCN-O2-NEXT: require<machine-loops> ; GCN-O2-NEXT: phi-node-elimination ; GCN-O2-NEXT: si-lower-control-flow +; GCN-O2-NEXT: amdgpu-private-object-vgprs ; GCN-O2-NEXT: two-address-instruction ; GCN-O2-NEXT: register-coalescer ; GCN-O2-NEXT: rename-independent-subregs @@ -419,6 +422,7 @@ ; GCN-O3-NEXT: require<machine-loops> ; GCN-O3-NEXT: phi-node-elimination ; GCN-O3-NEXT: si-lower-control-flow +; GCN-O3-NEXT: amdgpu-private-object-vgprs ; GCN-O3-NEXT: two-address-instruction ; GCN-O3-NEXT: register-coalescer ; GCN-O3-NEXT: rename-independent-subregs diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll index 1fd95e2289480..0cdd53c890d32 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -49,11 +49,13 @@ ; GCN-O0-NEXT: Lower uses of LDS variables from non-kernel functions ; GCN-O0-NEXT: FunctionPass Manager ; GCN-O0-NEXT: Expand Atomic instructions +; GCN-O0-NEXT: Dominator Tree Construction +; GCN-O0-NEXT: Natural Loop Information +; GCN-O0-NEXT: AMDGPU VGPR Allocate ; GCN-O0-NEXT: Remove unreachable blocks from the CFG ; GCN-O0-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining) ; GCN-O0-NEXT: Scalarize Masked Memory Intrinsics ; GCN-O0-NEXT: Expand reduction intrinsics -; GCN-O0-NEXT: Dominator Tree Construction ; GCN-O0-NEXT: AMDGPU Lower Kernel Arguments ; GCN-O0-NEXT: Lower buffer fat pointer operations to buffer resources ; GCN-O0-NEXT: FunctionPass Manager @@ -116,6 +118,7 @@ ; GCN-O0-NEXT: AMDGPU Lower Idx Ops ; GCN-O0-NEXT: Eliminate PHI nodes for register allocation ; GCN-O0-NEXT: SI Lower control flow pseudo instructions +; GCN-O0-NEXT: AMDGPU Def/use private object VGPRs ; GCN-O0-NEXT: Two-Address instruction pass ; GCN-O0-NEXT: MachineDominator Tree Construction ; GCN-O0-NEXT: Slot index numbering @@ -364,6 +367,7 @@ ; GCN-O1-NEXT: SI Optimize VGPR LiveRange ; GCN-O1-NEXT: Eliminate PHI nodes for register allocation ; GCN-O1-NEXT: SI Lower control flow pseudo instructions +; GCN-O1-NEXT: AMDGPU Def/use private object VGPRs ; GCN-O1-NEXT: Two-Address instruction pass ; GCN-O1-NEXT: Slot index numbering ; GCN-O1-NEXT: Live Interval Analysis @@ -686,6 +690,7 @@ ; GCN-O1-OPTS-NEXT: SI Optimize VGPR LiveRange ; GCN-O1-OPTS-NEXT: Eliminate PHI nodes for register allocation ; GCN-O1-OPTS-NEXT: SI Lower control flow pseudo instructions +; GCN-O1-OPTS-NEXT: AMDGPU Def/use private object VGPRs ; GCN-O1-OPTS-NEXT: Two-Address instruction pass ; GCN-O1-OPTS-NEXT: Slot index numbering ; GCN-O1-OPTS-NEXT: Live Interval Analysis @@ -1013,6 +1018,7 @@ ; GCN-O2-NEXT: SI Optimize VGPR LiveRange ; GCN-O2-NEXT: Eliminate PHI nodes for register allocation ; GCN-O2-NEXT: SI Lower control flow pseudo instructions +; GCN-O2-NEXT: AMDGPU Def/use private object VGPRs ; GCN-O2-NEXT: Two-Address instruction pass ; GCN-O2-NEXT: Slot index numbering ; GCN-O2-NEXT: Live Interval Analysis @@ -1354,6 +1360,7 @@ ; GCN-O3-NEXT: SI Optimize VGPR LiveRange ; GCN-O3-NEXT: Eliminate PHI nodes for register allocation ; GCN-O3-NEXT: SI Lower control flow pseudo instructions +; GCN-O3-NEXT: AMDGPU Def/use private object VGPRs ; GCN-O3-NEXT: Two-Address instruction pass ; GCN-O3-NEXT: Slot index numbering ; GCN-O3-NEXT: Live Interval Analysis _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
