llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-amdgpu Author: Gheorghe-Teodor Bercea (doru1004) <details> <summary>Changes</summary> Allocate and reserve registers for address space 13 objects --- Patch is 70.78 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/214248.diff 22 Files Affected: - (modified) llvm/include/llvm/IR/IntrinsicsAMDGPU.td (+9) - (modified) llvm/lib/Target/AMDGPU/AMDGPU.h (+22-1) - (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp (+33-1) - (modified) llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h (+2) - (modified) llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp (+14-1) - (modified) llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h (+19) - (modified) llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp (+26) - (modified) llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h (+19) - (modified) llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def (+2) - (added) llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp (+339) - (modified) llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp (+140-12) - (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp (+20-3) - (modified) llvm/lib/Target/AMDGPU/CMakeLists.txt (+1) - (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+57) - (modified) llvm/lib/Target/AMDGPU/SIISelLowering.h (+1) - (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+5-2) - (modified) llvm/lib/Target/AMDGPU/SIInstructions.td (+19) - (added) llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-across-call.ll (+70) - (added) llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-allocate.ll (+133) - (added) llvm/test/CodeGen/AMDGPU/AddressSpaceVGPR/as-vgpr-object-regalloc.ll (+178) - (modified) llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll (+4) - (modified) llvm/test/CodeGen/AMDGPU/llc-pipeline.ll (+8-1) ``````````diff diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 21882e247c027..1048f99752643 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -4292,4 +4292,13 @@ def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic< /// incoming block. def int_amdgcn_dead: DefaultAttrsIntrinsic<[llvm_any_ty], [], [IntrNoMem]>; + +/// Backend-only intrinsics analogous to llvm.lifetime.{start,end} for marking +/// the lifetimes of allocas in the VGPR address space. +def int_amdgcn_vgpr_lifetime_start : + DefaultAttrsIntrinsic<[], [llvm_anyptr_ty], + [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; +def int_amdgcn_vgpr_lifetime_end : + DefaultAttrsIntrinsic<[], [llvm_anyptr_ty], + [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index 84791f744cf25..4393269ffabc3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -226,6 +226,9 @@ extern char &AMDGPUMarkLastScratchLoadID; void initializeAMDGPULowerIdxOpsLegacyPass(PassRegistry &); extern char &AMDGPULowerIdxOpsID; +void initializeAMDGPUPrivateObjectVGPRsLegacyPass(PassRegistry &); +extern char &AMDGPUPrivateObjectVGPRsID; + void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &); extern char &SILowerSGPRSpillsLegacyID; @@ -266,7 +269,7 @@ void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &); extern char &AMDGPUPreloadKernelArgumentsLegacyID; // Passes common to R600 and SI -FunctionPass *createAMDGPUPromoteAlloca(); +FunctionPass *createAMDGPUPromoteAlloca(CodeGenOptLevel OptLevel); void initializeAMDGPUPromoteAllocaPass(PassRegistry&); extern char &AMDGPUPromoteAllocaID; @@ -279,6 +282,17 @@ struct AMDGPUPromoteAllocaPass TargetMachine &TM; }; +/// Only allocates objects in the VGPR ("as memory") address space, which is +/// required for functionality; used in place of AMDGPUPromoteAllocaPass when +/// optimizations are disabled. +struct AMDGPUVGPRAllocatePass : PassInfoMixin<AMDGPUVGPRAllocatePass> { + AMDGPUVGPRAllocatePass(TargetMachine &TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + +private: + TargetMachine &TM; +}; + struct AMDGPUPromoteAllocaToVectorPass : OptionalPassInfoMixin<AMDGPUPromoteAllocaToVectorPass> { AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {} @@ -447,6 +461,13 @@ class AMDGPULowerIdxOpsPass MachineFunctionAnalysisManager &MFAM); }; +class AMDGPUPrivateObjectVGPRsPass + : public RequiredPassInfoMixin<AMDGPUPrivateObjectVGPRsPass> { +public: + PreservedAnalyses run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM); +}; + class SIInsertWaitcntsPass : public RequiredPassInfoMixin<SIInsertWaitcntsPass> { public: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 0f88afbf20e37..5ba8c02f00b20 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -986,7 +986,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .widenScalarToNextPow2(0, 32) .clampMaxNumElements(0, S32, 16); - getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr}); + // An alloca in the VGPR ("as memory") address space is custom-lowered to the + // address it was allocated; see legalizeFrameIndexVGPR. + getActionDefinitionsBuilder(G_FRAME_INDEX) + .legalFor({PrivatePtr}) + .customFor({VGPRPtr}); // If the amount is divergent, we have to do a wave reduction to get the // maximum value, so this is expanded during RegBankSelect. @@ -2362,6 +2366,8 @@ bool AMDGPULegalizerInfo::legalizeCustom( switch (MI.getOpcode()) { case TargetOpcode::G_ADDRSPACE_CAST: return legalizeAddrSpaceCast(MI, MRI, B); + case TargetOpcode::G_FRAME_INDEX: + return legalizeFrameIndexVGPR(MI, B); case TargetOpcode::G_INTRINSIC_ROUNDEVEN: return legalizeFroundeven(MI, MRI, B); case TargetOpcode::G_FCEIL: @@ -3521,6 +3527,22 @@ static LLT widenToNextPowerOf2(LLT Ty) { /// legal G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE (or their _BITS forms for /// sub-dword accesses) indexed by the pointer's dword offset (pointer >> 2). /// Parallels the SelectionDAG LowerLoadStoreVGPR. +// The address of an object in the VGPR ("as memory") address space is where +// AMDGPUPromoteAlloca placed it, recorded on the alloca. Parallels the +// SelectionDAG SITargetLowering::lowerFrameIndex. +bool AMDGPULegalizerInfo::legalizeFrameIndexVGPR(MachineInstr &MI, + MachineIRBuilder &B) const { + MachineFunction &MF = B.getMF(); + int FI = MI.getOperand(1).getIndex(); + const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI); + assert(Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR); + + const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(*Alloca); + B.buildConstant(MI.getOperand(0), MD.getAddress()); + MI.eraseFromParent(); + return true; +} + static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, MachineInstr &MI) { MachineIRBuilder &B = Helper.MIRBuilder; MachineRegisterInfo &MRI = *B.getMRI(); @@ -8942,6 +8964,16 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, MI.eraseFromParent(); return true; } + case Intrinsic::amdgcn_vgpr_lifetime_start: + case Intrinsic::amdgcn_vgpr_lifetime_end: { + assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!"); + unsigned Opc = IntrID == Intrinsic::amdgcn_vgpr_lifetime_start + ? AMDGPU::VGPR_LIFETIME_START + : AMDGPU::VGPR_LIFETIME_END; + B.buildInstr(Opc).cloneMemRefs(MI); + MI.eraseFromParent(); + return true; + } case Intrinsic::amdgcn_cooperative_atomic_load_32x4B: case Intrinsic::amdgcn_cooperative_atomic_load_16x8B: case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index 30fd16930e6ae..1cc9119575731 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -41,6 +41,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo { MachineRegisterInfo &MRI, MachineIRBuilder &B) const; + bool legalizeFrameIndexVGPR(MachineInstr &MI, MachineIRBuilder &B) const; + bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp index b109d0f5b2389..c8af1830378dd 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp @@ -16,6 +16,8 @@ #include "AMDGPU.h" #include "AMDGPUAsmPrinter.h" #include "AMDGPUMachineFunctionInfo.h" +#include "AMDGPUMachineInstrs.h" +#include "AMDGPUMemoryUtils.h" #include "MCTargetDesc/AMDGPUInstPrinter.h" #include "MCTargetDesc/AMDGPUMCExpr.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" @@ -435,8 +437,19 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) { } if (MI->isMetaInstruction()) { - if (isVerbose()) + if (isVerbose()) { + if (const auto *Marker = dyn_cast<AMDGPUMI::VGPRLifetimeInst>(MI)) { + const auto &MD = + AMDGPU::AllocatedVGPRsMetadata::get(Marker->getObject()); + unsigned Begin = MD.getAddress() / 4; + unsigned End = (MD.getAddress() + MD.getSize() - 1) / 4; + OutStreamer->emitRawComment( + Twine(" VGPR lifetime ") + (Marker->isStart() ? "start" : "end") + + ": v[" + Twine(Begin) + ":" + Twine(End) + "]"); + return; + } OutStreamer->emitRawComment(" meta instruction"); + } return; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h index e99594062871a..5497eb74de3de 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineInstrs.h @@ -74,6 +74,25 @@ class VStoreIdxInst : public VLoadStoreIdxInst { } }; +// Wrapper for the markers bounding the lifetime of an object in the VGPR "as +// memory" (address space 13) address space. The object is named by the memory +// operand, which is the only thing these carry until +// AMDGPUPrivateObjectVGPRs gives them the registers it occupies. +class VGPRLifetimeInst : public MachineInstr { +public: + bool isStart() const { return getOpcode() == AMDGPU::VGPR_LIFETIME_START; } + + const AllocaInst &getObject() const { + return *cast<AllocaInst>((*memoperands_begin())->getValue()); + } + + static bool classof(const MachineInstr *MI) { + unsigned Opc = MI->getOpcode(); + return Opc == AMDGPU::VGPR_LIFETIME_START || + Opc == AMDGPU::VGPR_LIFETIME_END; + } +}; + } // end namespace AMDGPUMI } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp index 7186ef6a5a827..7614fc23fac40 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp @@ -31,6 +31,32 @@ Align getAlign(const DataLayout &DL, const GlobalVariable *GV) { GV->getValueType()); } +AllocatedVGPRsMetadata &AllocatedVGPRsMetadata::get(const AllocaInst &Alloca) { + return *cast<AllocatedVGPRsMetadata>( + Alloca.getMetadata("amdgpu.allocated.vgprs")); +} + +unsigned AllocatedVGPRsMetadata::getAddress() const { + return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(0))->getValue()) + ->getZExtValue(); +} + +unsigned AllocatedVGPRsMetadata::getSize() const { + return cast<ConstantInt>(cast<ConstantAsMetadata>(getOperand(1))->getValue()) + ->getZExtValue(); +} + +bool AllocatedVGPRsMetadata::classof(const MDNode *N) { + if (N->getNumOperands() != 2) + return false; + for (int I = 0; I != 2; ++I) { + auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(I)); + if (!C || !isa<ConstantInt>(C->getValue())) + return false; + } + return true; +} + void copyMetadataForWidenedLoad(LoadInst &Dest, const LoadInst &Source) { SmallVector<std::pair<unsigned, MDNode *>, 8> MD; Source.getAllMetadata(MD); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h index f81b677051561..db04c254feaeb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.h @@ -12,11 +12,13 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/IR/Metadata.h" namespace llvm { struct Align; class AAResults; +class AllocaInst; class DataLayout; class GlobalVariable; class LoadInst; @@ -106,6 +108,23 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA); bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA, AAResults *AA); +/// Convenience wrapper for !amdgpu.allocated.vgprs metadata, which records +/// where in the VGPR ("as memory") address space an alloca was placed. +/// +/// NOTE: Using `isa` with this class is not very meaningful. +class AllocatedVGPRsMetadata : public MDNode { +public: + static AllocatedVGPRsMetadata &get(const AllocaInst &Alloca); + + /// Get the indicated (byte) address for the VGPR allocation. + unsigned getAddress() const; + + /// Get the size (in bytes) of the VGPR allocation. + unsigned getSize() const; + + static bool classof(const MDNode *N); +}; + } // end namespace AMDGPU } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def index 534d97df145c0..e5385d189b94b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def +++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def @@ -73,6 +73,7 @@ FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass()) FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass()) FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes", AMDGPUUnifyDivergentExitNodesPass()) +FUNCTION_PASS("amdgpu-vgpr-allocate", AMDGPUVGPRAllocatePass(*this)) FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass()) FUNCTION_PASS("si-annotate-control-flow", SIAnnotateControlFlowPass(*static_cast<const GCNTargetMachine *>(this))) FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass()) @@ -124,6 +125,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-agpr-copy-mfma", AMDGPURewriteAGPRCopyMFMA MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass()) MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass()) MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass()) +MACHINE_FUNCTION_PASS("amdgpu-private-object-vgprs", AMDGPUPrivateObjectVGPRsPass()) MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass()) MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass()) MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass()) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp new file mode 100644 index 0000000000000..750988d23c2cd --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUPrivateObjectVGPRs.cpp @@ -0,0 +1,339 @@ +//===- AMDGPUPrivateObjectVGPRs.cpp - Private object VGPRs ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// Mark the physical VGPRs an object in the VGPR ("as memory") address space +/// was allocated to as used, so that register allocation does not hand them to +/// anything else where the object is live: +/// +/// * Add implicit use/def operands to the VGPR_LIFETIME_{START,END} pseudos +/// * Add the VGPRs to basic block live-ins +/// +/// The allocation itself is recorded on the alloca as !amdgpu.allocated.vgprs +/// by AMDGPUPromoteAlloca, and reaches this pass through the memory operand of +/// the lifetime markers. +// +//===----------------------------------------------------------------------===// + +#include "AMDGPU.h" +#include "AMDGPUMachineInstrs.h" +#include "AMDGPUMemoryUtils.h" +#include "GCNSubtarget.h" +#include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/ADT/SmallBitVector.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/LiveVariables.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/Instructions.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/MathExtras.h" + +using namespace llvm; + +#define DEBUG_TYPE "amdgpu-private-object-vgprs" + +using ObjectRegs = SmallVector<MCPhysReg, 50>; + +namespace { + +class AMDGPUPrivateObjectVGPRsImpl { +public: + AMDGPUPrivateObjectVGPRsImpl(const MachineFunction &MF, LiveIntervals *LIS) + : TII(MF.getSubtarget<GCNSubtarget>().getInstrInfo()), LIS(LIS) {} + + bool run(MachineFunction &MF); + +private: + struct AllocaBBInfo { + const AllocaInst *Alloca = nullptr; + bool LiveIn = false; + bool Starts = false; + bool Ends = false; + }; + + ObjectRegs computeObjectRegs(const AllocaInst &Alloca) const; + + const SIInstrInfo *TII; + LiveIntervals *LIS; + + DenseMap<const AllocaInst *, std::pair<ObjectRegs, MachineMemOperand *>> + AllocaObjectRegs; +}; + +} // end anonymous namespace + +// The registers an object occupies: one per dword, starting at the register its +// byte address in the address space names. +ObjectRegs AMDGPUPrivateObjectVGPRsImpl::computeObjectRegs( + const AllocaInst &Alloca) const { + const auto &MD = AMDGPU::AllocatedVGPRsMetadata::get(Alloca); + unsigned Offset = MD.getAddress(); + assert(Offset % 4 == 0 && "Object is not dword aligned"); + + // An object that does not fill its last register still owns the whole of it. + unsigned BaseRegIdx = Offset / 4; + unsigned NumRegs = divideCeil(MD.getSize(), 4); + + ObjectRegs Regs; + Regs.reserve(NumRegs); + for (unsigned I : seq(NumRegs)) + Regs.push_back(AMDGPU::VGPR_32RegClass.getRegister(BaseRegIdx + I)); + + return Regs; +} + +bool AMDGPUPrivateObjectVGPRsImpl::run(MachineFunction &MF) { + // Sort basic blocks in reverse post-order for the live-out/live-in + // propagation. + DenseMap<MachineBasicBlock *, unsigned> BlockToIndex; + SmallVector<MachineBasicBlock *> IndexToBlock; + ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin()); + for (MachineBasicBlock *MBB : RPOT) { + BlockToIndex[MBB] = IndexToBlock.size(); + IndexToBlock.push_back(MBB); + } + + // Fixed-point iteration to determine basic block live-ins. + // + // The first pass of the fixed-point iteration also scans instructions. + SmallVector<SmallVector<AllocaBBInfo>> BBInfos(IndexToBlock.size()); + SmallBitVector Worklist(IndexToBlock.size()); + bool Changed = false; + + for (bool Dirty = true, FirstPass = true; Dirty; FirstPass = false) { + Dirty = false; + + for (auto [MBBI, MBB] : enumerate(IndexToBlock)) { + auto &BBI = BBInfos[MBBI]; + + // During the first outer iteration, augment VGPR_LIFETIME_{START,END} + // with implicit operands and record the initial per-basic block + // information to compute live-ins. + if (FirstPass) { + for (MachineInstr &MI : *MBB) { + if (MI.getOpcode() != AMDGPU::VGPR_LIFETIME_START && + MI.getOpcode() != AMDGPU::VGPR_LIFETIME_END) + continue; + + bool IsStart = MI.getOpcode() == AMDGPU::VGPR_LIFETIME_START; + MachineMemOperand *MMO = *MI.memoperands_begin(); + const auto *Alloca = cast<AllocaInst>(MMO->getValue()); + + auto ObjRegsIt = AllocaObjectRegs.find(Alloca); + if (ObjRegsIt == AllocaObjectRegs.end()) + ObjRegsIt = AllocaObjectRegs + .try_emplace(Alloca, computeObjectRegs(*Alloca), MMO) + .first; + + // The object comes into existence at the start and dies at the end. + for (MCPhysReg Reg : ObjRegsIt->second.first) + MI.addOperand(MachineOperand::CreateReg( + Reg, /*isDef=*/IsStart, /*isImp=*/true, /*isKill=*/!IsStart)); + + auto It = find_if(BBI, [&](const AllocaBBInfo &Info) { + return Info.Alloca == Alloca; + }); + if (It == BBI.end()) { + BBI.push_back({Alloca, false, false, false}); + It = std::prev(BBI.end()); + } + It->Starts = IsStart; + It->Ends = !IsStart; + + Changed = true; + } + } else { + if (!Worklist[MBBI]) + continue; + Worklist[MBBI] = false; + } + + // Propagate live-outs into successors. + for (const auto &ABBI : BBI) { + if (!((ABBI.LiveIn && !ABBI.Ends) || ABBI.Starts)) + continue; + + const ObjectRegs &Regs = AllocaObjectRegs.at(ABBI.Alloca).first; + for (MachineBasicBlock *Succ : MBB->successors()) { + unsigned SuccI = BlockToIndex.at(Succ); + auto &SuccBBI = BBInfos[SuccI]; + auto It = find_if(SuccBBI, [&](const AllocaBBInfo &Info) { + return Info.Alloca == ABBI.Alloca; + }); + + bool Update = false; + if (It == SuccBBI.end()) { + SuccBBI.push_back({ABBI.Alloca, true, false, false}); + It = std::prev(SuccBBI.end()); + Update = true; + } else if (!It->LiveIn) { + It->LiveIn = true; + Update = true; + } + + if (!Update) + continue; + + // We are... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/214248 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
