https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/229630
>From 99f38f4cbe010f3f791b20cb0100e6ca9a7cd351 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Tue, 6 Oct 2026 06:14:26 -0700 Subject: [PATCH 1/4] [RegAllocFast] Fold foldable inline asm operands under register pressure An inline asm register operand marked foldable (from an "rm" constraint) may be replaced with a stack slot when the register allocator runs out of registers. The greedy allocator does that when it spills the value. The fast allocator can't: it assigns the operands of an instruction one at a time, and folding replaces the instruction. So it reported "inline assembly requires more registers than available" instead. Before allocating a block, estimate from each inline asm's own operands whether they fit in registers, and fold as many foldable registers as needed to make them fit, so the common case without pressure still gets a register. Values that only live across the asm don't count, since the allocator spills them when it needs their registers. The estimate follows allocateInstruction(): - First all defs get distinct registers, avoiding physreg defs; then all uses, along with the defs still occupied while the uses are read (early-clobber and tied defs, see isLiveThroughDef()), get distinct registers, avoiding physreg uses and early-clobber physreg defs. - A register counts against another register class as many times as one of its registers overlaps that class's registers, so e.g. 32-bit "rm" operands compete with 64-bit operands for the same registers. - A value used by both an "rm" and an "r" operand isn't folded, since that frees no register. - An input tied to a def keeps a register of its own when the allocator lowers the tie itself and the input is read later in the block. Kill flags are optional, so a missing one doesn't mean the input lives on. Folding a register folds all of its operands at once, with one store before the asm and one reload after it. It uses the register's own spill slot, so that a block that reloads the register, such as an INLINEASM_BR's indirect target, which the reload after the asm doesn't reach, finds the value the asm wrote. The whole thing is skipped for functions without inline asm. Nothing in tree marks an inline asm register operand foldable yet, so the tests set the "foldable" flag in MIR. This follows the shape of the unmerged #74344, "[RegAllocFast] fold foldable inline asm", with a pressure estimate instead of folding unconditionally, per the review there. Assisted-by: Claude Opus 5.5 --- llvm/lib/CodeGen/RegAllocFast.cpp | 320 ++++++++++++ .../regallocfast-inline-asm-fold-pressure.mir | 479 ++++++++++++++++++ .../X86/regallocfast-inline-asm-fold.mir | 80 +++ 3 files changed, 879 insertions(+) create mode 100644 llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-pressure.mir create mode 100644 llvm/test/CodeGen/X86/regallocfast-inline-asm-fold.mir diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp index 50a4f15612082..2617f929e677f 100644 --- a/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/llvm/lib/CodeGen/RegAllocFast.cpp @@ -20,6 +20,10 @@ /// Where the target enables it, TwoAddressInstructionPass is left out of the /// pipeline: this pass lowers tied operands and expands REG_SEQUENCE and /// INSERT_SUBREG itself. +/// +/// An inline asm register operand that may be folded to memory (from an "rm" +/// constraint) is folded to a stack slot before its block is allocated, if +/// the asm's register operands wouldn't fit in registers otherwise. // //===----------------------------------------------------------------------===// @@ -64,6 +68,8 @@ using namespace llvm; STATISTIC(NumStores, "Number of stores added"); STATISTIC(NumLoads, "Number of loads added"); STATISTIC(NumCoalesced, "Number of copies coalesced"); +STATISTIC(NumInlineAsmFolds, + "Number of inline asm registers folded to stack slots"); static RegisterRegAlloc fastRegAlloc("fast", "fast register allocator", createFastRegisterAllocator); @@ -422,6 +428,12 @@ class RegAllocFastImpl { bool mayBeSpillFromInlineAsmBr(const MachineInstr &MI) const; + void foldInlineAsmOperands(MachineBasicBlock &MBB); + bool tiedUseNeedsOwnReg(const MachineInstr &MI, unsigned UseIdx) const; + void selectInlineAsmRegsToFold(const MachineInstr &MI, + SmallVectorImpl<Register> &ToFold) const; + void foldInlineAsmReg(MachineInstr *&MI, Register Reg); + void dumpState() const; }; @@ -1960,6 +1972,309 @@ void RegAllocFastImpl::handleBundle(MachineInstr &MI) { } } +/// Return true if lowerTiedUse() will give the input \p UseIdx of \p MI, tied +/// to a virtual def, a register of its own to copy into the def's register. +/// It does when the value already has a register below MI, i.e. it's read +/// further down the block (a read in another block reloads it from its stack +/// slot), and when the value can't simply move into the def's register. Kill +/// flags are optional here, so a missing one doesn't mean the value lives on. +bool RegAllocFastImpl::tiedUseNeedsOwnReg(const MachineInstr &MI, + unsigned UseIdx) const { + const MachineOperand &MO = MI.getOperand(UseIdx); + const MachineOperand &DefMO = MI.getOperand(MI.findTiedOperandIdx(UseIdx)); + Register Reg = MO.getReg(); + if (MO.getSubReg() || + !MRI->getRegClass(Reg)->hasSubClassEq(MRI->getRegClass(DefMO.getReg()))) + return true; + + // Another read of the value by MI may get it a register first. + if (any_of(MI.all_uses(), [&](const MachineOperand &O) { + return &O != &MO && O.getReg() == Reg; + })) + return true; + + if (MO.isKill()) + return false; + + // Only scan the rest of the block if the value is read elsewhere in it, + // which is rare for the input of a tied operand. + const MachineBasicBlock *MBB = MI.getParent(); + if (none_of(MRI->use_nodbg_instructions(Reg), [&](const MachineInstr &UseMI) { + return &UseMI != &MI && UseMI.getParent() == MBB; + })) + return false; + + return any_of( + make_range(std::next(MachineBasicBlock::const_iterator(MI)), MBB->end()), + [&](const MachineInstr &Other) { + return !Other.isDebugInstr() && Other.readsVirtualRegister(Reg); + }); +} + +/// Choose which foldable ("rm") registers of the inline asm \p MI to fold to +/// stack slots so that the rest of its register operands fit, and add them to +/// \p ToFold. +/// +/// The greedy allocator folds such an operand when it runs out of registers. +/// This allocator can't: it assigns the operands of an instruction one at a +/// time, and folding replaces the instruction. So estimate up front, from the +/// asm's own operands, whether they fit. Values that only live across the asm +/// don't count, since this allocator spills them when it needs their +/// registers. The estimate follows allocateInstruction(): first all defs get +/// distinct registers, avoiding physreg defs; then all uses, along with the +/// defs still occupied while the uses are read (early-clobber and tied defs), +/// get distinct registers, avoiding physreg uses and early-clobber physreg +/// defs. Folding a register removes it from both. +/// +/// A register counts against another register class as many times as one of +/// its registers overlaps registers of that class, e.g. once for GR64 against +/// GR32. That, and keeping physreg uses out of the defs' registers, errs +/// toward folding more than necessary rather than running out of registers. +void RegAllocFastImpl::selectInlineAsmRegsToFold( + const MachineInstr &MI, SmallVectorImpl<Register> &ToFold) const { + // The virtual registers of MI's register operands. + struct AsmReg { + Register Reg; + const TargetRegisterClass *RC; + // Needs a register while the defs are assigned. + bool InDefs = false; + // Needs a register while the uses are read. + bool InUses = false; + // Inputs tied to this def that keep registers of their own, to be copied + // from (see lowerTiedUse()). Folding the def folds them too. + unsigned TiedInputs = 0; + // Read by an operand that isn't tied to its def. + bool HasUntiedUse = false; + // Every untied operand of the register can be folded. + bool Foldable = true; + bool Folded = false; + }; + SmallVector<AsmReg, 8> Regs; + auto getAsmReg = [&](Register Reg) -> AsmReg & { + for (AsmReg &R : Regs) + if (R.Reg == Reg) + return R; + + return Regs.emplace_back(AsmReg{Reg, MRI->getRegClass(Reg)}); + }; + + // Physical registers taken while the defs, and while the uses, are assigned. + SmallVector<MCRegister, 8> DefTaken; + SmallVector<MCRegister, 8> UseTaken; + + for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); + I != E; ++I) { + const MachineOperand &MO = MI.getOperand(I); + if (!MO.isReg() || !MO.getReg() || (MO.isUse() && MO.isUndef())) + continue; + + Register Reg = MO.getReg(); + if (Reg.isPhysical()) { + DefTaken.push_back(Reg); + if (MO.isUse() || isLiveThroughDef(MI, MO)) + UseTaken.push_back(Reg); + + continue; + } + + if (!shouldAllocateRegister(Reg)) + continue; + + // A tied use is read through its def's register. Unless that is its own + // register (once tied operands have been rewritten), it can also need a + // register of its own to copy from (see tiedUseNeedsOwnReg()), and always + // does when the def is a fixed register, which might not be allocatable. + if (MO.isUse() && MO.isTied()) { + Register DefReg = MI.getOperand(MI.findTiedOperandIdx(I)).getReg(); + if (Reg == DefReg) + continue; + + if (DefReg.isVirtual()) { + if (tiedUseNeedsOwnReg(MI, I)) + ++getAsmReg(DefReg).TiedInputs; + } else { + AsmReg &R = getAsmReg(Reg); + R.InUses = true; + R.Foldable = false; + } + + continue; + } + + AsmReg &R = getAsmReg(Reg); + R.Foldable &= MI.mayFoldInlineAsmRegOp(I); + if (MO.isUse()) { + R.InUses = true; + R.HasUntiedUse = true; + } else { + R.InDefs = true; + R.InUses |= isLiveThroughDef(MI, MO); + } + } + + // The asm's own read of a register it also writes can't share the def's + // stack slot unless that's how the two are tied, so only fold a register + // the asm either reads or writes. + for (AsmReg &R : Regs) + R.Foldable &= !(R.InDefs && R.HasUntiedUse); + + // Fold within one register class at a time, in operand order. Registers + // folded for one class no longer count against the next. + SmallVector<const TargetRegisterClass *, 4> Done; + for (const AsmReg &Candidate : Regs) { + const TargetRegisterClass *RC = Candidate.RC; + if (!Candidate.Foldable || is_contained(Done, RC)) + continue; + + Done.push_back(RC); + + ArrayRef<MCPhysReg> Order = RegClassInfo.getOrder(RC); + auto countFree = [&](ArrayRef<MCRegister> Taken) -> unsigned { + return count_if(Order, [&](MCPhysReg PhysReg) { + return none_of( + Taken, [&](MCRegister T) { return TRI->regsOverlap(PhysReg, T); }); + }); + }; + unsigned DefsFree = countFree(DefTaken); + unsigned UsesFree = countFree(UseTaken); + + // How many registers of RC one register of each AsmReg can take. + SmallVector<unsigned, 8> Weight; + for (const AsmReg &R : Regs) { + if (RC->hasSubClassEq(R.RC) || R.RC->hasSubClassEq(RC)) { + Weight.push_back(1); + continue; + } + + unsigned Max = 0; + for (MCPhysReg PhysReg : RegClassInfo.getOrder(R.RC)) + Max = std::max<unsigned>(Max, count_if(Order, [&](MCPhysReg Other) { + return TRI->regsOverlap(PhysReg, Other); + })); + + Weight.push_back(Max); + } + + while (true) { + unsigned DefsNeeded = 0; + unsigned UsesNeeded = 0; + for (auto [R, W] : zip(Regs, Weight)) { + if (R.Folded) + continue; + + DefsNeeded += R.InDefs * W; + UsesNeeded += (R.InUses + R.TiedInputs) * W; + } + + bool DefsShort = DefsNeeded > DefsFree; + bool UsesShort = UsesNeeded > UsesFree; + if (!DefsShort && !UsesShort) + break; + + // Fold the first register that relieves the most phases that are short. + AsmReg *Best = nullptr; + unsigned BestRelief = 0; + for (auto [R, W] : zip(Regs, Weight)) { + if (R.Folded || !R.Foldable || !W) + continue; + + unsigned Relief = + (DefsShort && R.InDefs) + (UsesShort && (R.InUses || R.TiedInputs)); + if (Relief > BestRelief) { + Best = &R; + BestRelief = Relief; + } + } + + // Without anything left to fold, allocation reports the error. + if (!Best) + break; + + Best->Folded = true; + ToFold.push_back(Best->Reg); + } + } +} + +/// Fold the operands of \p Reg in the inline asm \p MI to Reg's stack slot, +/// storing the value the asm reads there before it, and reloading Reg from it +/// afterward if the asm writes Reg. \p MI is replaced by the new instruction. +void RegAllocFastImpl::foldInlineAsmReg(MachineInstr *&MI, Register Reg) { + // Fold every operand of Reg. A tied use goes along with its def, and reads + // the def's slot, which therefore needs the tied input's value, whichever + // register that is. + SmallVector<unsigned, 2> Ops; + Register ReadReg; + bool Writes = false; + for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI->getNumOperands(); + I != E; ++I) { + const MachineOperand &MO = MI->getOperand(I); + if (!MO.isReg() || MO.getReg() != Reg) + continue; + + if (MO.isUse()) { + if (!MO.isTied()) + Ops.push_back(I); + + if (MO.readsReg()) + ReadReg = Reg; + + continue; + } + + Ops.push_back(I); + Writes |= !MO.isDead(); + if (MO.isTied()) { + const MachineOperand &Use = MI->getOperand(MI->findTiedOperandIdx(I)); + if (Use.readsReg()) + ReadReg = Use.getReg(); + } + } + + // Use Reg's own spill slot: a block that reloads Reg, such as an + // INLINEASM_BR's indirect target, which the reload after the asm doesn't + // reach, then finds the value the asm wrote. + MachineBasicBlock &MBB = *MI->getParent(); + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + int FI = getStackSpaceFor(Reg); + MachineInstr *CopyMI = nullptr; + MachineInstr *NewMI = TII->foldMemoryOperand(*MI, Ops, FI, CopyMI); + assert(NewMI && !CopyMI && "inline asm register should fold"); + + // NewMI is inserted before MI. The allocator works out the kill flags. + if (ReadReg) { + TII->storeRegToStackSlot(MBB, NewMI->getIterator(), ReadReg, + /*isKill=*/false, FI, RC, ReadReg); + ++NumStores; + } + + if (Writes) { + TII->loadRegFromStackSlot(MBB, std::next(NewMI->getIterator()), Reg, FI, RC, + Reg); + ++NumLoads; + } + + ++NumInlineAsmFolds; + + MI->eraseFromParent(); + MI = NewMI; +} + +/// Fold the operands of each inline asm in \p MBB that won't fit in registers +/// (see selectInlineAsmRegsToFold()). +void RegAllocFastImpl::foldInlineAsmOperands(MachineBasicBlock &MBB) { + for (MachineInstr &MI : make_early_inc_range(MBB)) { + if (!MI.isInlineAsm()) + continue; + + SmallVector<Register, 4> ToFold; + selectInlineAsmRegsToFold(MI, ToFold); + MachineInstr *AsmMI = &MI; + for (Register Reg : ToFold) + foldInlineAsmReg(AsmMI, Reg); + } +} + void RegAllocFastImpl::allocateBasicBlock(MachineBasicBlock &MBB) { this->MBB = &MBB; LLVM_DEBUG(dbgs() << "\nAllocating " << MBB); @@ -1973,6 +2288,11 @@ void RegAllocFastImpl::allocateBasicBlock(MachineBasicBlock &MBB) { Coalesced.clear(); + // Folding replaces an inline asm, which allocateInstruction() can't do + // partway through, so fold the operands that won't fit in registers first. + if (MBB.getParent()->hasInlineAsm()) + foldInlineAsmOperands(MBB); + // Lowering a tied operand inserts a copy ahead of MI. Its registers are // already assigned, so visiting it would evict what still lives in the // source. diff --git a/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-pressure.mir b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-pressure.mir new file mode 100644 index 0000000000000..87f7239f56bdb --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-pressure.mir @@ -0,0 +1,479 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=regallocfast \ +# RUN: -verify-machineinstrs -verify-regalloc %s -o - | FileCheck %s + +# The fast register allocator folds foldable ("rm") inline asm operands to +# stack slots when the asm's register operands wouldn't fit in registers +# otherwise, and only then. + +--- +# The asm clobbers rax/rbx/rbp/r10-r15, leaving six of the 15 GPRs for its +# register operands, and its six "r" inputs take all of them. The "=&rm" +# output can't share a register with an input, so it's folded to a stack slot. +name: test_rm_output_pressure +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } + - { id: 1, class: gr64 } + - { id: 2, class: gr64 } + - { id: 3, class: gr64 } + - { id: 4, class: gr64 } + - { id: 5, class: gr64 } + - { id: 6, class: gr64_norex2 } + - { id: 7, class: gr64_norex2 } + - { id: 8, class: gr64_norex2 } + - { id: 9, class: gr64_norex2 } + - { id: 10, class: gr64_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: test_rm_output_pressure + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM &"# $0 $1 $2 $3 $4 $5 $6", sideeffect maystore attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64_NOREX2, killed renamable $rdi, reguse:GR64_NOREX2, killed renamable $rsi, reguse:GR64_NOREX2, killed renamable $rdx, reguse:GR64_NOREX2, killed renamable $rcx, reguse:GR64_NOREX2, killed renamable $r8, reguse:GR64_NOREX2, killed renamable $r9, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.0) + ; CHECK-NEXT: renamable $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0, killed $rax + %5:gr64 = COPY $r9 + %4:gr64 = COPY $r8 + %3:gr64 = COPY $rcx + %2:gr64 = COPY $rdx + %1:gr64 = COPY $rsi + %0:gr64 = COPY $rdi + %7:gr64_norex2 = COPY %0 + %8:gr64_norex2 = COPY %1 + %9:gr64_norex2 = COPY %2 + %10:gr64_norex2 = COPY %3 + %11:gr64_norex2 = COPY %4 + %12:gr64_norex2 = COPY %5 + INLINEASM &"# $0 $1 $2 $3 $4 $5 $6", sideeffect attdialect, regdef-ec:GR64_NOREX2 foldable, def early-clobber %6, reguse:GR64_NOREX2, %7, reguse:GR64_NOREX2, %8, reguse:GR64_NOREX2, %9, reguse:GR64_NOREX2, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# One value passed to two "rm" operands: spilling it means folding both +# operands into its stack slot at once. +name: test_rm_same_value_twice_pressure +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } + - { id: 1, class: gr64 } + - { id: 2, class: gr64 } + - { id: 3, class: gr64 } + - { id: 4, class: gr64 } + - { id: 5, class: gr64 } + - { id: 6, class: gr64_norex2 } + - { id: 7, class: gr64_norex2 } + - { id: 8, class: gr64_norex2 } + - { id: 9, class: gr64_norex2 } + - { id: 10, class: gr64_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } + - { id: 13, class: gr64_norex2 } + - { id: 14, class: gr64 } +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: test_rm_same_value_twice_pressure + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $rax = COPY killed $rdi + ; CHECK-NEXT: renamable $rdi = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rax :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64_NOREX2, killed renamable $rsi, reguse:GR64_NOREX2, killed renamable $rdx, reguse:GR64_NOREX2, killed renamable $rcx, reguse:GR64_NOREX2, killed renamable $r8, reguse:GR64_NOREX2, killed renamable $r9, reguse:GR64_NOREX2, killed renamable $rdi, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0 + %5:gr64 = COPY $r9 + %4:gr64 = COPY $r8 + %3:gr64 = COPY $rcx + %2:gr64 = COPY $rdx + %1:gr64 = COPY $rsi + %0:gr64 = COPY $rdi + %14:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + %6:gr64_norex2 = COPY %0 + %8:gr64_norex2 = COPY %1 + %9:gr64_norex2 = COPY %2 + %10:gr64_norex2 = COPY %3 + %11:gr64_norex2 = COPY %4 + %12:gr64_norex2 = COPY %5 + %13:gr64_norex2 = COPY %14 + INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7", sideeffect attdialect, reguse:GR64_NOREX2 foldable, %6, reguse:GR64_NOREX2 foldable, %6, reguse:GR64_NOREX2, %8, reguse:GR64_NOREX2, %9, reguse:GR64_NOREX2, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, reguse:GR64_NOREX2, %13, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + RET 0 +... +--- +# Folding the "rm" input moves the operands after it, including the input +# tied to the "=r" output, whose tie must survive. +name: test_rm_before_tied_pressure +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } + - { id: 1, class: gr64 } + - { id: 2, class: gr64 } + - { id: 3, class: gr64 } + - { id: 4, class: gr64 } + - { id: 5, class: gr64 } + - { id: 6, class: gr64_norex2 } + - { id: 7, class: gr64_norex2 } + - { id: 8, class: gr64_norex2 } + - { id: 9, class: gr64_norex2 } + - { id: 10, class: gr64_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } + - { id: 13, class: gr64_norex2 } + - { id: 14, class: gr64 } +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: test_rm_before_tied_pressure + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.2) + ; CHECK-NEXT: renamable $rax = COPY $rdi + ; CHECK-NEXT: $rdi = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2) + ; CHECK-NEXT: renamable $rcx = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rax :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7", sideeffect mayload attdialect, regdef:GR64_NOREX2, def renamable $rcx, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64_NOREX2, killed renamable $rsi, reguse:GR64_NOREX2, killed renamable $rdx, reguse:GR64_NOREX2, killed renamable $rdi, reguse:GR64_NOREX2, killed renamable $r8, reguse:GR64_NOREX2, killed renamable $r9, reguse tiedto:$0, renamable $rcx(tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1) + ; CHECK-NEXT: $rax = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: RET 0, killed $rax + %5:gr64 = COPY $r9 + %4:gr64 = COPY $r8 + %3:gr64 = COPY $rcx + %2:gr64 = COPY $rdx + %1:gr64 = COPY $rsi + %0:gr64 = COPY $rdi + %14:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + %7:gr64_norex2 = COPY %0 + %8:gr64_norex2 = COPY %1 + %9:gr64_norex2 = COPY %2 + %10:gr64_norex2 = COPY %3 + %11:gr64_norex2 = COPY %4 + %12:gr64_norex2 = COPY %5 + %13:gr64_norex2 = COPY %14 + %6:gr64_norex2 = COPY %13 + INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7", sideeffect attdialect, regdef:GR64_NOREX2, def %6, reguse:GR64_NOREX2 foldable, %7, reguse:GR64_NOREX2, %8, reguse:GR64_NOREX2, %9, reguse:GR64_NOREX2, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, reguse tiedto:$0, %6(tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# Two read-write "+rm" operands: folding one tied pair must keep the other +# pair tied. +name: test_two_tied_rm_pressure +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } + - { id: 1, class: gr64 } + - { id: 2, class: gr64 } + - { id: 3, class: gr64 } + - { id: 4, class: gr64 } + - { id: 5, class: gr64 } + - { id: 6, class: gr64_norex2 } + - { id: 7, class: gr64_norex2 } + - { id: 8, class: gr64_norex2 } + - { id: 9, class: gr64_norex2 } + - { id: 10, class: gr64_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } + - { id: 13, class: gr64_norex2 } + - { id: 14, class: gr64_norex2 } + - { id: 15, class: gr64 } +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: test_two_tied_rm_pressure + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1) + ; CHECK-NEXT: renamable $rcx = COPY killed $rdx + ; CHECK-NEXT: renamable $rdx = COPY $rsi + ; CHECK-NEXT: $rsi = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: renamable $rax = COPY killed $rdi + ; CHECK-NEXT: renamable $rdi = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, renamable $rax :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"# $0 $1 $2 $3 $4 $5 $6", sideeffect mayload maystore attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, regdef:GR64_NOREX2 foldable, def renamable $rdx, reguse:GR64_NOREX2, killed renamable $rcx, reguse:GR64_NOREX2, killed renamable $rsi, reguse:GR64_NOREX2, killed renamable $r8, reguse:GR64_NOREX2, killed renamable $r9, reguse:GR64_NOREX2, killed renamable $rdi, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse tiedto:$1, renamable $rdx(tied-def 9), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.0) + ; CHECK-NEXT: renamable $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0, killed $rax, killed $rdx + %5:gr64 = COPY $r9 + %4:gr64 = COPY $r8 + %3:gr64 = COPY $rcx + %2:gr64 = COPY $rdx + %1:gr64 = COPY $rsi + %0:gr64 = COPY $rdi + %15:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + %8:gr64_norex2 = COPY %2 + %9:gr64_norex2 = COPY %3 + %10:gr64_norex2 = COPY %4 + %11:gr64_norex2 = COPY %5 + %12:gr64_norex2 = COPY %15 + %13:gr64_norex2 = COPY %0 + %14:gr64_norex2 = COPY %1 + %6:gr64_norex2 = COPY %13 + %7:gr64_norex2 = COPY %14 + INLINEASM &"# $0 $1 $2 $3 $4 $5 $6", sideeffect attdialect, regdef:GR64_NOREX2 foldable, def %6, regdef:GR64_NOREX2 foldable, def %7, reguse:GR64_NOREX2, %8, reguse:GR64_NOREX2, %9, reguse:GR64_NOREX2, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, reguse tiedto:$0, %6(tied-def 3), reguse tiedto:$1, %7(tied-def 5), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + $rax = COPY %6 + $rdx = COPY %7 + RET 0, $rax, $rdx +... +--- +# Like test_rm_output_pressure, but the output isn't early-clobber, so it can +# share a register with an input and nothing needs to be folded. +name: test_rm_output_shares_input_register +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } + - { id: 1, class: gr64 } + - { id: 2, class: gr64 } + - { id: 3, class: gr64 } + - { id: 4, class: gr64 } + - { id: 5, class: gr64 } + - { id: 6, class: gr64_norex2 } + - { id: 7, class: gr64_norex2 } + - { id: 8, class: gr64_norex2 } + - { id: 9, class: gr64_norex2 } + - { id: 10, class: gr64_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: test_rm_output_shares_input_register + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM &"# $0 $1 $2 $3 $4 $5 $6", sideeffect attdialect, regdef:GR64_NOREX2 foldable, def renamable $rcx, reguse:GR64_NOREX2, killed renamable $rdi, reguse:GR64_NOREX2, killed renamable $rsi, reguse:GR64_NOREX2, killed renamable $rdx, reguse:GR64_NOREX2, killed renamable $rcx, reguse:GR64_NOREX2, killed renamable $r8, reguse:GR64_NOREX2, killed renamable $r9, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.0) + ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0, killed $rax + %5:gr64 = COPY $r9 + %4:gr64 = COPY $r8 + %3:gr64 = COPY $rcx + %2:gr64 = COPY $rdx + %1:gr64 = COPY $rsi + %0:gr64 = COPY $rdi + %7:gr64_norex2 = COPY %0 + %8:gr64_norex2 = COPY %1 + %9:gr64_norex2 = COPY %2 + %10:gr64_norex2 = COPY %3 + %11:gr64_norex2 = COPY %4 + %12:gr64_norex2 = COPY %5 + INLINEASM &"# $0 $1 $2 $3 $4 $5 $6", sideeffect attdialect, regdef:GR64_NOREX2 foldable, def %6, reguse:GR64_NOREX2, %7, reguse:GR64_NOREX2, %8, reguse:GR64_NOREX2, %9, reguse:GR64_NOREX2, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# Eleven inputs for the ten GPRs left after the clobbers: one "rm" input has +# to be folded even though the 32-bit inputs alone would fit, because each +# 64-bit input takes a 32-bit register too. +name: test_rm_mixed_widths_pressure +tracksRegLiveness: true +registers: + - { id: 0, class: gr32 } + - { id: 1, class: gr32 } + - { id: 2, class: gr32 } + - { id: 3, class: gr32 } + - { id: 4, class: gr32 } + - { id: 5, class: gr64 } + - { id: 6, class: gr32_norex2 } + - { id: 7, class: gr32_norex2 } + - { id: 8, class: gr32_norex2 } + - { id: 9, class: gr32_norex2 } + - { id: 10, class: gr32_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } + - { id: 13, class: gr64_norex2 } + - { id: 14, class: gr64_norex2 } + - { id: 15, class: gr64_norex2 } + - { id: 16, class: gr64_norex2 } + - { id: 17, class: gr64 } + - { id: 18, class: gr64 } + - { id: 19, class: gr64 } + - { id: 20, class: gr64 } + - { id: 21, class: gr64 } +fixedStack: + - { id: 0, offset: 32, size: 8, alignment: 16, isImmutable: true } + - { id: 1, offset: 24, size: 8, alignment: 8, isImmutable: true } + - { id: 2, offset: 16, size: 8, alignment: 16, isImmutable: true } + - { id: 3, offset: 8, size: 8, alignment: 8, isImmutable: true } + - { id: 4, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $edi, $esi, $edx, $ecx, $r8d, $r9 + + ; CHECK-LABEL: name: test_rm_mixed_widths_pressure + ; CHECK: liveins: $edi, $esi, $edx, $ecx, $r8d, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $ebx = COPY killed $edi + ; CHECK-NEXT: renamable $rax = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: renamable $rdi = MOV64rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.1) + ; CHECK-NEXT: renamable $r10 = MOV64rm %fixed-stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.2, align 16) + ; CHECK-NEXT: renamable $r11 = MOV64rm %fixed-stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.3) + ; CHECK-NEXT: renamable $r15 = MOV64rm %fixed-stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.4, align 16) + ; CHECK-NEXT: MOV32mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $ebx :: (store (s32) into %stack.0) + ; CHECK-NEXT: INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7 $8 $9 $10", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR32_NOREX2 foldable, killed renamable $esi, reguse:GR32_NOREX2 foldable, killed renamable $edx, reguse:GR32_NOREX2 foldable, killed renamable $ecx, reguse:GR32_NOREX2 foldable, killed renamable $r8d, reguse:GR64_NOREX2, killed renamable $r9, reguse:GR64_NOREX2, killed renamable $rax, reguse:GR64_NOREX2, killed renamable $rdi, reguse:GR64_NOREX2, killed renamable $r10, reguse:GR64_NOREX2, killed renamable $r11, reguse:GR64_NOREX2, killed renamable $r15, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14 :: (load (s32) from %stack.0) + ; CHECK-NEXT: RET 0 + %5:gr64 = COPY $r9 + %4:gr32 = COPY $r8d + %3:gr32 = COPY $ecx + %2:gr32 = COPY $edx + %1:gr32 = COPY $esi + %0:gr32 = COPY $edi + %17:gr64 = MOV64rm %fixed-stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.4, align 16) + %18:gr64 = MOV64rm %fixed-stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.3) + %19:gr64 = MOV64rm %fixed-stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.2, align 16) + %20:gr64 = MOV64rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.1) + %21:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + %6:gr32_norex2 = COPY %0 + %7:gr32_norex2 = COPY %1 + %8:gr32_norex2 = COPY %2 + %9:gr32_norex2 = COPY %3 + %10:gr32_norex2 = COPY %4 + %11:gr64_norex2 = COPY %5 + %12:gr64_norex2 = COPY %17 + %13:gr64_norex2 = COPY %18 + %14:gr64_norex2 = COPY %19 + %15:gr64_norex2 = COPY %20 + %16:gr64_norex2 = COPY %21 + INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7 $8 $9 $10", sideeffect attdialect, reguse:GR32_NOREX2 foldable, %6, reguse:GR32_NOREX2 foldable, %7, reguse:GR32_NOREX2 foldable, %8, reguse:GR32_NOREX2 foldable, %9, reguse:GR32_NOREX2 foldable, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, reguse:GR64_NOREX2, %13, reguse:GR64_NOREX2, %14, reguse:GR64_NOREX2, %15, reguse:GR64_NOREX2, %16, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14 + RET 0 +... +--- +# Seven values for six GPRs. Folding the "rm" operand of %6 wouldn't free a +# register, since the "r" operand needs %6 in one anyway, so fold %8's. +name: test_rm_and_r_same_value_pressure +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } + - { id: 1, class: gr64 } + - { id: 2, class: gr64 } + - { id: 3, class: gr64 } + - { id: 4, class: gr64 } + - { id: 5, class: gr64 } + - { id: 6, class: gr64_norex2 } + - { id: 7, class: gr64_norex2 } + - { id: 8, class: gr64_norex2 } + - { id: 9, class: gr64_norex2 } + - { id: 10, class: gr64_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } + - { id: 13, class: gr64_norex2 } + - { id: 14, class: gr64 } +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: test_rm_and_r_same_value_pressure + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $rax = COPY killed $rsi + ; CHECK-NEXT: renamable $rsi = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rax :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7", sideeffect mayload attdialect, reguse:GR64_NOREX2 foldable, killed renamable $rdi, reguse:GR64_NOREX2, renamable $rdi, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64_NOREX2, killed renamable $rdx, reguse:GR64_NOREX2, killed renamable $rcx, reguse:GR64_NOREX2, killed renamable $r8, reguse:GR64_NOREX2, killed renamable $r9, reguse:GR64_NOREX2, killed renamable $rsi, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0 + %5:gr64 = COPY $r9 + %4:gr64 = COPY $r8 + %3:gr64 = COPY $rcx + %2:gr64 = COPY $rdx + %1:gr64 = COPY $rsi + %0:gr64 = COPY $rdi + %14:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + %6:gr64_norex2 = COPY %0 + %8:gr64_norex2 = COPY %1 + %9:gr64_norex2 = COPY %2 + %10:gr64_norex2 = COPY %3 + %11:gr64_norex2 = COPY %4 + %12:gr64_norex2 = COPY %5 + %13:gr64_norex2 = COPY %14 + INLINEASM &"# $0 $1 $2 $3 $4 $5 $6 $7", sideeffect attdialect, reguse:GR64_NOREX2 foldable, %6, reguse:GR64_NOREX2, %6, reguse:GR64_NOREX2 foldable, %8, reguse:GR64_NOREX2, %9, reguse:GR64_NOREX2, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, reguse:GR64_NOREX2, %13, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + RET 0 +... +--- +# Like test_rm_output_pressure, with an INLINEASM_BR. The reload of %7 after +# the asm only covers the fallthrough path: on the indirect path, %7 must come +# from the slot the asm wrote, which has to be %7's own spill slot, so that +# the indirect block's ordinary reload of %7 finds it. +name: test_callbr_rm_indirect_use +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } + - { id: 1, class: gr64_nosp } + - { id: 2, class: gr64 } + - { id: 3, class: gr64 } + - { id: 4, class: gr64 } + - { id: 5, class: gr64 } + - { id: 6, class: gr64 } + - { id: 7, class: gr64_norex2 } + - { id: 8, class: gr64_norex2 } + - { id: 9, class: gr64_norex2 } + - { id: 10, class: gr64_norex2 } + - { id: 11, class: gr64_norex2 } + - { id: 12, class: gr64_norex2 } + - { id: 13, class: gr64_norex2 } + - { id: 14, class: gr64_norex2 } + - { id: 15, class: gr64 } + - { id: 16, class: gr32 } + - { id: 17, class: gr64 } +body: | + ; CHECK-LABEL: name: test_callbr_rm_indirect_use + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000), %bb.2(0x00000000) + ; CHECK-NEXT: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.1) + ; CHECK-NEXT: INLINEASM_BR &"# $0 $1 $2 $3 $4 $5 $6 ${7:l}", sideeffect maystore attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64_NOREX2, killed renamable $rdi, reguse:GR64_NOREX2, killed renamable $rsi, reguse:GR64_NOREX2, killed renamable $rdx, reguse:GR64_NOREX2, killed renamable $rcx, reguse:GR64_NOREX2, killed renamable $r8, reguse:GR64_NOREX2, killed renamable $r9, imm, %bb.2, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.0) + ; CHECK-NEXT: renamable $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed $rax :: (store (s64) into %stack.0) + ; CHECK-NEXT: JMP_1 %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: renamable $eax = MOV32r0 implicit-def dead $eflags + ; CHECK-NEXT: renamable $rax = SUBREG_TO_REG killed renamable $eax, %subreg.sub_32bit + ; CHECK-NEXT: RET 0, killed $rax + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2 (inlineasm-br-indirect-target): + ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: $rcx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: renamable $rax = LEA64r killed renamable $rax, 1, killed renamable $rcx, 0, $noreg + ; CHECK-NEXT: RET 0, killed $rax + bb.0: + successors: %bb.1(0x80000000), %bb.2(0x00000000) + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + %6:gr64 = COPY $r9 + %5:gr64 = COPY $r8 + %4:gr64 = COPY $rcx + %3:gr64 = COPY $rdx + %2:gr64 = COPY $rsi + %1:gr64_nosp = COPY $rdi + %8:gr64_norex2 = COPY %1 + %9:gr64_norex2 = COPY %2 + %10:gr64_norex2 = COPY %3 + %11:gr64_norex2 = COPY %4 + %12:gr64_norex2 = COPY %5 + %13:gr64_norex2 = COPY %6 + INLINEASM_BR &"# $0 $1 $2 $3 $4 $5 $6 ${7:l}", sideeffect attdialect, regdef-ec:GR64_NOREX2 foldable, def early-clobber %7, reguse:GR64_NOREX2, %8, reguse:GR64_NOREX2, %9, reguse:GR64_NOREX2, %10, reguse:GR64_NOREX2, %11, reguse:GR64_NOREX2, %12, reguse:GR64_NOREX2, %13, imm, %bb.2, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + JMP_1 %bb.1 + + bb.1: + %16:gr32 = MOV32r0 implicit-def dead $eflags + %17:gr64 = SUBREG_TO_REG killed %16, %subreg.sub_32bit + $rax = COPY %17 + RET 0, $rax + + bb.2 (inlineasm-br-indirect-target): + %15:gr64 = LEA64r %7, 1, %1, 0, $noreg + $rax = COPY %15 + RET 0, $rax +... + diff --git a/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold.mir b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold.mir new file mode 100644 index 0000000000000..870277e0fe2f1 --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold.mir @@ -0,0 +1,80 @@ +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=regallocfast \ +# RUN: -verify-machineinstrs %s -o - | FileCheck %s + +# The fast register allocator folds a foldable ("rm") inline asm operand to a +# stack slot when the asm's register operands wouldn't fit otherwise. The +# clobbers below leave six GPRs. The tied operands haven't been rewritten, so +# a tied def and its input are different virtual registers: the allocator +# lowers the tie itself, copying the input into the def's register if the +# input stays live. + +--- +# Five inputs plus the tied pair fit: the input dies, so the def takes over its +# register. +name: tied_input_dies +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: tied_input_dies + ; CHECK-NOT: MOV64mr + ; CHECK: INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def renamable [[REG:\$[a-z0-9]+]], {{.*}}, reguse tiedto:$0, killed renamable [[REG]](tied-def 3) + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %6:gr64, reguse:GR64, killed %1, reguse:GR64, killed %2, reguse:GR64, killed %3, reguse:GR64, killed %4, reguse:GR64, killed %5, reguse tiedto:$0, killed %0(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# The same without kill flags, which are optional: nothing reads the input +# after the asm, so it still dies there. +name: tied_input_dies_no_kill_flag +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: tied_input_dies_no_kill_flag + ; CHECK-NOT: MOV64mr + ; CHECK: INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def renamable [[REG:\$[a-z0-9]+]], {{.*}}, reguse tiedto:$0, killed renamable [[REG]](tied-def 3) + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %6:gr64, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse tiedto:$0, %0(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# The same, except that the input is used after the asm, so it keeps its own +# register while it's copied into the def's: seven registers. Fold the tied +# pair instead, storing the input's value to the def's stack slot first. +name: tied_input_lives +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: tied_input_lives + ; CHECK: MOV64mr %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, renamable [[INPUT:\$[a-z0-9]+]] + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, {{.*}}, mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, clobber, + ; CHECK-NEXT: renamable [[DEF:\$[a-z0-9]+]] = MOV64rm %stack.[[SLOT]], 1, $noreg, 0, $noreg + ; CHECK-NEXT: ADD64rr killed renamable [[DEF]], killed renamable [[INPUT]] + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %6:gr64, reguse:GR64, killed %1, reguse:GR64, killed %2, reguse:GR64, killed %3, reguse:GR64, killed %4, reguse:GR64, killed %5, reguse tiedto:$0, %0(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + %7:gr64 = ADD64rr %6, killed %0, implicit-def dead $eflags + $rax = COPY %7 + RET 0, $rax +... >From bae38633e447706f4c712dc7ec24400c1f4363da Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Thu, 15 Oct 2026 19:20:41 -0700 Subject: [PATCH 2/4] [RegAllocFast] Address review: don't recount a tied input's untied reads When one value is passed to both a tied operand and an untied one, e.g. "=rm"(r) : "r"(v), "0"(v), with an explicit -regalloc=fast above -O0 (so TwoAddress doesn't run and the ties reach RegAllocFast), the fold estimate counted three registers where lowerTiedUse() uses one. Tied uses are allocated first, so the value moves into the def's register and the untied read is rewritten to it. tiedUseNeedsOwnReg() still assumed another read of the value could get it a register first, and the untied read was counted on its own as well, so the asm's output was folded to the stack needlessly. Model the tied input itself: while one of its tied defs is in a register, it needs no register of its own unless tiedUseNeedsOwnReg() says so (only an early-clobber def now makes another read matter), and its untied reads share whichever register it has. Once its tied defs are folded, the untied reads need a register again, so choose what to fold by counting again with each candidate folded, and fold both halves when neither alone helps. Assisted-by: Claude Opus 5.5 --- llvm/lib/CodeGen/RegAllocFast.cpp | 88 ++++++-- ...egallocfast-inline-asm-fold-tied-input.mir | 209 ++++++++++++++++++ 2 files changed, 278 insertions(+), 19 deletions(-) create mode 100644 llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp index 2617f929e677f..1d97dbe3f8df0 100644 --- a/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/llvm/lib/CodeGen/RegAllocFast.cpp @@ -1978,6 +1978,10 @@ void RegAllocFastImpl::handleBundle(MachineInstr &MI) { /// further down the block (a read in another block reloads it from its stack /// slot), and when the value can't simply move into the def's register. Kill /// flags are optional here, so a missing one doesn't mean the value lives on. +/// +/// MI's other reads of the value don't give it a register first: tied uses +/// are allocated before untied ones, and a value that moved into one tied +/// def's register is copied from there into another's. bool RegAllocFastImpl::tiedUseNeedsOwnReg(const MachineInstr &MI, unsigned UseIdx) const { const MachineOperand &MO = MI.getOperand(UseIdx); @@ -1987,8 +1991,10 @@ bool RegAllocFastImpl::tiedUseNeedsOwnReg(const MachineInstr &MI, !MRI->getRegClass(Reg)->hasSubClassEq(MRI->getRegClass(DefMO.getReg()))) return true; - // Another read of the value by MI may get it a register first. - if (any_of(MI.all_uses(), [&](const MachineOperand &O) { + // An early-clobber def overwrites its register before MI's other reads of + // the value, so they need it in a register of its own. + if (DefMO.isEarlyClobber() && + any_of(MI.all_uses(), [&](const MachineOperand &O) { return &O != &MO && O.getReg() == Reg; })) return true; @@ -2026,6 +2032,12 @@ bool RegAllocFastImpl::tiedUseNeedsOwnReg(const MachineInstr &MI, /// get distinct registers, avoiding physreg uses and early-clobber physreg /// defs. Folding a register removes it from both. /// +/// The input of a tied def usually moves into the def's register, and the +/// asm's untied reads of it follow, so it needs no register of its own while +/// that def isn't folded. Otherwise (see tiedUseNeedsOwnReg()) it keeps one +/// register, both to copy from and for its untied reads, until it and its +/// tied defs are all folded. +/// /// A register counts against another register class as many times as one of /// its registers overlaps registers of that class, e.g. once for GR64 against /// GR32. That, and keeping physreg uses out of the defs' registers, errs @@ -2040,9 +2052,12 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( bool InDefs = false; // Needs a register while the uses are read. bool InUses = false; - // Inputs tied to this def that keep registers of their own, to be copied - // from (see lowerTiedUse()). Folding the def folds them too. - unsigned TiedInputs = 0; + // The virtual defs this register is the tied input of. Folding such a def + // folds the tied use too. + SmallVector<Register, 1> TiedDefs = {}; + // As a tied input, keeps a register of its own instead of moving into a + // tied def's register (see tiedUseNeedsOwnReg()). + bool TiedNeedsOwnReg = false; // Read by an operand that isn't tied to its def. bool HasUntiedUse = false; // Every untied operand of the register can be folded. @@ -2090,8 +2105,10 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( continue; if (DefReg.isVirtual()) { - if (tiedUseNeedsOwnReg(MI, I)) - ++getAsmReg(DefReg).TiedInputs; + bool NeedsOwnReg = tiedUseNeedsOwnReg(MI, I); + AsmReg &R = getAsmReg(Reg); + R.TiedDefs.push_back(DefReg); + R.TiedNeedsOwnReg |= NeedsOwnReg; } else { AsmReg &R = getAsmReg(Reg); R.InUses = true; @@ -2115,8 +2132,13 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( // The asm's own read of a register it also writes can't share the def's // stack slot unless that's how the two are tied, so only fold a register // the asm either reads or writes. - for (AsmReg &R : Regs) - R.Foldable &= !(R.InDefs && R.HasUntiedUse); + // Folding a register only folds its untied operands, so a register the asm + // only reads through tied uses frees nothing. + for (AsmReg &R : Regs) { + R.Foldable &= !(R.InDefs && R.HasUntiedUse) && (R.InDefs || R.HasUntiedUse); + // A tied input the asm also writes isn't modeled; give it a register. + R.TiedNeedsOwnReg |= R.InDefs; + } // Fold within one register class at a time, in operand order. Registers // folded for one class no longer count against the next. @@ -2155,32 +2177,60 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( Weight.push_back(Max); } - while (true) { - unsigned DefsNeeded = 0; - unsigned UsesNeeded = 0; + // Count the registers of RC needed while the defs, and while the uses, are + // assigned, given which registers are folded so far. + auto countNeeded = [&](unsigned &DefsNeeded, unsigned &UsesNeeded) { + DefsNeeded = UsesNeeded = 0; for (auto [R, W] : zip(Regs, Weight)) { - if (R.Folded) - continue; + DefsNeeded += (!R.Folded && R.InDefs) * W; + bool NeedsReg = !R.Folded && R.InUses; + if (!R.TiedDefs.empty()) { + // While a tied def is in a register, the input is read from there, + // possibly copied from a register of its own. + bool InDefReg = any_of(R.TiedDefs, [&](Register Def) { + return !find_if(Regs, [&](const AsmReg &D) { + return D.Reg == Def; + })->Folded; + }); + NeedsReg = R.TiedNeedsOwnReg ? NeedsReg || InDefReg + : NeedsReg && !InDefReg; + } - DefsNeeded += R.InDefs * W; - UsesNeeded += (R.InUses + R.TiedInputs) * W; + UsesNeeded += NeedsReg * W; } + }; + while (true) { + unsigned DefsNeeded, UsesNeeded; + countNeeded(DefsNeeded, UsesNeeded); bool DefsShort = DefsNeeded > DefsFree; bool UsesShort = UsesNeeded > UsesFree; if (!DefsShort && !UsesShort) break; // Fold the first register that relieves the most phases that are short. + // Folding a tied def can give its input a register back, so count again + // rather than just dropping the register's own needs. AsmReg *Best = nullptr; unsigned BestRelief = 0; for (auto [R, W] : zip(Regs, Weight)) { if (R.Folded || !R.Foldable || !W) continue; - unsigned Relief = - (DefsShort && R.InDefs) + (UsesShort && (R.InUses || R.TiedInputs)); - if (Relief > BestRelief) { + unsigned DefsAfter, UsesAfter; + R.Folded = true; + countNeeded(DefsAfter, UsesAfter); + R.Folded = false; + unsigned Relief = (DefsShort && DefsAfter < DefsNeeded) + + (UsesShort && UsesAfter < UsesNeeded); + // Folding only one of a tied def and an input that shares its register + // relieves nothing; folding both does. So, failing any relief, fold + // the first register that lowers the total without making a short + // phase worse. + bool Worse = (DefsShort && DefsAfter > DefsNeeded) || + (UsesShort && UsesAfter > UsesNeeded); + bool Lowers = DefsAfter + UsesAfter < DefsNeeded + UsesNeeded; + if (Relief > BestRelief || (!Best && !Worse && Lowers)) { Best = &R; BestRelief = Relief; } diff --git a/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir new file mode 100644 index 0000000000000..d5ba8661c42c9 --- /dev/null +++ b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir @@ -0,0 +1,209 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=regallocfast \ +# RUN: -verify-machineinstrs %s -o - | FileCheck %s + +# The fast register allocator folds a foldable ("rm") inline asm operand to a +# stack slot only when the asm's register operands wouldn't fit otherwise. The +# clobbers below leave six GPRs. The tied operands haven't been rewritten, so +# the allocator lowers each tie itself: tied uses are allocated first, and the +# input moves into its def's register, where the asm's untied reads of the +# same value find it too. These tests check that such an input isn't counted +# again for its untied reads, and that nothing is folded when it all fits. +# +# Each test is derived from the C function and inline asm statement in its +# comment, compiled with -regalloc=fast, where CLOBBERS is: +# +# "rax", "rbx", "rbp", "r10", "r11", "r12", "r13", "r14", "r15" + +--- +# One value read through the tied "0" input and an untied "r" input, plus five +# other inputs: the def's register holds the value for both reads, so the six +# registers fit. +# +# long tied_and_untied(long v, long a, long b, long c, long d, long e) { +# long r; +# asm volatile("" : "=rm"(r) : "r"(v), "r"(a), "r"(b), "r"(c), "r"(d), +# "r"(e), "0"(v) : CLOBBERS); +# return r; +# } +name: tied_and_untied +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-LABEL: name: tied_and_untied + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def renamable $rdi, reguse:GR64, renamable $rdi, reguse:GR64, killed renamable $rsi, reguse:GR64, killed renamable $rdx, reguse:GR64, killed renamable $rcx, reguse:GR64, killed renamable $r8, reguse:GR64, killed renamable $r9, reguse tiedto:$0, killed renamable $rdi(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0, killed $rax + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %6:gr64, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse tiedto:$0, %0(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# The same value is also read after the asm, so it keeps a register of its own +# to copy into the def's register, and the untied read uses one of those two: +# with four other inputs, six registers fit. +# +# long tied_and_untied_live_after(long v, long a, long b, long c, long d) { +# long r; +# asm volatile("" : "=rm"(r) : "r"(v), "r"(a), "r"(b), "r"(c), "r"(d), +# "0"(v) : CLOBBERS); +# return r + v; +# } +name: tied_and_untied_live_after +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8 + ; CHECK-LABEL: name: tied_and_untied_live_after + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1) + ; CHECK-NEXT: renamable $r9 = COPY $r8 + ; CHECK-NEXT: $r8 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: $rcx = COPY $rdi + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def renamable $rcx, reguse:GR64, renamable $rcx, reguse:GR64, killed renamable $rsi, reguse:GR64, killed renamable $rdx, reguse:GR64, killed renamable $r8, reguse:GR64, killed renamable $r9, reguse tiedto:$0, renamable $rcx(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.0) + ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: renamable $rax = ADD64rr killed renamable $rax, killed renamable $rdi, implicit-def dead $eflags + ; CHECK-NEXT: RET 0, killed $rax + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %6:gr64, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse tiedto:$0, %0(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + %7:gr64 = ADD64rr %6, %0, implicit-def dead $eflags + $rax = COPY %7 + RET 0, $rax +... +--- +# An early-clobber def overwrites its register before the untied read, so the +# value keeps a register of its own, shared by the copy and the untied read: +# with four other inputs, six registers fit. +# +# long tied_early_clobber(long v, long a, long b, long c, long d) { +# long r; +# asm volatile("" : "=&rm"(r) : "r"(v), "r"(a), "r"(b), "r"(c), "r"(d), +# "0"(v) : CLOBBERS); +# return r; +# } +name: tied_early_clobber +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8 + ; CHECK-LABEL: name: tied_early_clobber + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.2) + ; CHECK-NEXT: renamable $rax = COPY $rcx + ; CHECK-NEXT: $rcx = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rax :: (store (s64) into %stack.1) + ; CHECK-NEXT: renamable $r9 = COPY $r8 + ; CHECK-NEXT: $r8 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: $rdi = COPY $rcx + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef-ec:GR64 foldable, def early-clobber renamable $rdi, reguse:GR64, renamable $rcx, reguse:GR64, killed renamable $rsi, reguse:GR64, killed renamable $rdx, reguse:GR64, killed renamable $r8, reguse:GR64, killed renamable $r9, reguse tiedto:$0, killed renamable $rdi(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0, killed $rax + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + INLINEASM &"", sideeffect attdialect, regdef-ec:GR64 foldable, def early-clobber %6:gr64, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse tiedto:$0, %0(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# One value tied to two defs: it moves into the first def's register and is +# copied from there into the second's, so it needs no register of its own: +# with four other inputs, six registers fit. +# +# struct pair { long a, b; }; +# +# struct pair tied_twice(long v, long a, long b, long c, long d) { +# long r, s; +# asm volatile("" : "=rm"(r), "=rm"(s) : "r"(a), "r"(b), "r"(c), "r"(d), +# "0"(v), "1"(v) : CLOBBERS); +# return (struct pair){r, s}; +# } +name: tied_twice +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8 + ; CHECK-LABEL: name: tied_twice + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.2) + ; CHECK-NEXT: renamable $rax = COPY $rcx + ; CHECK-NEXT: $rcx = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rax :: (store (s64) into %stack.1) + ; CHECK-NEXT: renamable $r9 = COPY $r8 + ; CHECK-NEXT: $r8 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: $rdx = COPY $rdi + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def renamable $rdi, regdef:GR64 foldable, def renamable $rdx, reguse:GR64, killed renamable $rsi, reguse:GR64, killed renamable $rcx, reguse:GR64, killed renamable $r8, reguse:GR64, killed renamable $r9, reguse tiedto:$0, killed renamable $rdi(tied-def 3), reguse tiedto:$1, renamable $rdx(tied-def 5), clobber, implicit-def early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0, killed $rax, killed $rdx + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %6:gr64, regdef:GR64 foldable, def %7:gr64, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse tiedto:$0, %0(tied-def 3), reguse tiedto:$1, %0(tied-def 5), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + $rax = COPY %6 + $rdx = COPY %7 + RET 0, $rax, $rdx +... +--- +# Six other inputs leave no register for the shared one. Folding only the def +# or only the untied read frees nothing, since the other then needs the value +# in a register, so both are folded. +# +# long tied_and_untied_pressure(long v, long a, long b, long c, long d, +# long e, long f) { +# long r; +# asm volatile("" : "=rm"(r) : "rm"(v), "r"(a), "r"(b), "r"(c), "r"(d), +# "r"(e), "r"(f), "0"(v) : CLOBBERS); +# return r; +# } +name: tied_and_untied_pressure +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-LABEL: name: tied_and_untied_pressure + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $rax = COPY killed $rdi + ; CHECK-NEXT: renamable $rdi = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, renamable $rax :: (store (s64) into %stack.0) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, killed renamable $rax :: (store (s64) into %stack.1) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, mem:m, %stack.1, 1, $noreg, 0, $noreg, reguse:GR64, killed renamable $rsi, reguse:GR64, killed renamable $rdx, reguse:GR64, killed renamable $rcx, reguse:GR64, killed renamable $r8, reguse:GR64, killed renamable $r9, reguse:GR64, killed renamable $rdi, mem:m, %stack.0, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.0), (load (s64) from %stack.1) + ; CHECK-NEXT: renamable $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0, killed $rax + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %7:gr64, reguse:GR64 foldable, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, reguse tiedto:$0, %0(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + $rax = COPY %7 + RET 0, $rax +... >From 750966e240babdc1e2d8aacfb588558ce3385175 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Thu, 15 Oct 2026 19:37:45 -0700 Subject: [PATCH 3/4] [RegAllocFast] Reformat Assisted-by: Claude Opus 5.5 --- llvm/lib/CodeGen/RegAllocFast.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp index 1d97dbe3f8df0..48dd7f61cbb2e 100644 --- a/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/llvm/lib/CodeGen/RegAllocFast.cpp @@ -2192,8 +2192,8 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( return D.Reg == Def; })->Folded; }); - NeedsReg = R.TiedNeedsOwnReg ? NeedsReg || InDefReg - : NeedsReg && !InDefReg; + NeedsReg = + R.TiedNeedsOwnReg ? NeedsReg || InDefReg : NeedsReg && !InDefReg; } UsesNeeded += NeedsReg * W; >From ba3623d33fee6575433df245f08a0360cd59b620 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Thu, 22 Oct 2026 01:56:25 -0700 Subject: [PATCH 4/4] [RegAllocFast] Address review: don't fold registers whose slot can't hold the value Folding a register stores the value the asm reads into the register's slot, which can't work in two cases, so leave the register unfolded: - A def that is also the tied input of another def. Its slot would need both its old value, for that def, and its own tied input's value. The store put the old value there, so the asm read the wrong input. - An operand, or a def's tied input, that reads a subregister. The slot holds the whole register; storing a GR64 tied input for a GR32 def failed the machine verifier. ISel never emits either form. Also skip the analysis for inline asm without a foldable operand, which is most of it. Assisted-by: Claude Opus 5.5 --- llvm/lib/CodeGen/RegAllocFast.cpp | 20 ++++- ...egallocfast-inline-asm-fold-tied-input.mir | 79 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp index 48dd7f61cbb2e..60ae4876a1f32 100644 --- a/llvm/lib/CodeGen/RegAllocFast.cpp +++ b/llvm/lib/CodeGen/RegAllocFast.cpp @@ -32,6 +32,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/Sequence.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseSet.h" @@ -2044,6 +2045,11 @@ bool RegAllocFastImpl::tiedUseNeedsOwnReg(const MachineInstr &MI, /// toward folding more than necessary rather than running out of registers. void RegAllocFastImpl::selectInlineAsmRegsToFold( const MachineInstr &MI, SmallVectorImpl<Register> &ToFold) const { + // Most inline asm has no foldable operand, so skip the analysis below. + if (none_of(seq<unsigned>(InlineAsm::MIOp_FirstOperand, MI.getNumOperands()), + [&](unsigned I) { return MI.mayFoldInlineAsmRegOp(I); })) + return; + // The virtual registers of MI's register operands. struct AsmReg { Register Reg; @@ -2105,6 +2111,11 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( continue; if (DefReg.isVirtual()) { + // Folding the def stores the whole tied input into the def's slot, so + // an input read through a subregister rules that out. + if (MO.getSubReg()) + getAsmReg(DefReg).Foldable = false; + bool NeedsOwnReg = tiedUseNeedsOwnReg(MI, I); AsmReg &R = getAsmReg(Reg); R.TiedDefs.push_back(DefReg); @@ -2118,8 +2129,10 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( continue; } + // The slot holds the whole register, so a subregister operand isn't + // folded. AsmReg &R = getAsmReg(Reg); - R.Foldable &= MI.mayFoldInlineAsmRegOp(I); + R.Foldable &= MI.mayFoldInlineAsmRegOp(I) && !MO.getSubReg(); if (MO.isUse()) { R.InUses = true; R.HasUntiedUse = true; @@ -2134,9 +2147,12 @@ void RegAllocFastImpl::selectInlineAsmRegsToFold( // the asm either reads or writes. // Folding a register only folds its untied operands, so a register the asm // only reads through tied uses frees nothing. + // A tied input the asm also writes isn't modeled; give it a register, and + // don't fold it: its slot would need both its old value, for the def it's + // tied to, and the tied input of its own def. for (AsmReg &R : Regs) { R.Foldable &= !(R.InDefs && R.HasUntiedUse) && (R.InDefs || R.HasUntiedUse); - // A tied input the asm also writes isn't modeled; give it a register. + R.Foldable &= !(R.InDefs && !R.TiedDefs.empty()); R.TiedNeedsOwnReg |= R.InDefs; } diff --git a/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir index d5ba8661c42c9..5f73ce22cb7de 100644 --- a/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir +++ b/llvm/test/CodeGen/X86/regallocfast-inline-asm-fold-tied-input.mir @@ -207,3 +207,82 @@ body: | $rax = COPY %7 RET 0, $rax ... +--- +# A foldable def that is also the tied input of another def: folding it +# would need its slot to hold both its old value, for the "1" input, and the +# "0" input's value. So it stays in a register and gets the "0" input, the +# "=r" def gets its old value, and "rm" inputs are folded instead. ISel never +# emits this, since a def is a new SSA value, so the MIR redefines %0 by hand. +# +# asm volatile("" : "=rm"(a), "=r"(b) : "r"(c), "r"(d), "rm"(e), "rm"(f), +# "rm"(g), "0"(y), "1"(a) : CLOBBERS); +name: tied_def_also_tied_input +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-LABEL: name: tied_def_also_tied_input + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.1) + ; CHECK-NEXT: renamable $rdi = COPY $rcx + ; CHECK-NEXT: $rcx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: renamable $rax = COPY killed $r8 + ; CHECK-NEXT: renamable $r8 = MOV64ri 42 + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rax :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64 foldable, def renamable $rsi, regdef:GR64, def renamable $rcx, reguse:GR64, killed renamable $rdx, reguse:GR64, killed renamable $rdi, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64 foldable, killed renamable $r9, reguse:GR64 foldable, killed renamable $r8, reguse tiedto:$0, killed renamable $rsi(tied-def 3), reguse tiedto:$1, renamable $rcx(tied-def 5), clobber, implicit-def early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1) + ; CHECK-NEXT: $rax = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: renamable $rax = ADD64rr killed renamable $rax, killed renamable $rcx, implicit-def dead $eflags + ; CHECK-NEXT: RET 0, killed $rax + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + %6:gr64 = MOV64ri 42 + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %0:gr64, regdef:GR64, def %7:gr64, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64 foldable, %4, reguse:GR64 foldable, %5, reguse:GR64 foldable, %6, reguse tiedto:$0, %1(tied-def 3), reguse tiedto:$1, %0(tied-def 5), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + %8:gr64 = ADD64rr %0, %7, implicit-def dead $eflags + $rax = COPY %8 + RET 0, $rax +... +--- +# A foldable def whose tied input is read through a subregister: the slot +# would need the whole input, so the def stays in a register, and "rm" +# inputs are folded instead. ISel copies a tied input to a register of the +# def's class, so only MIR reads one through a subregister. +# +# asm volatile("" : "=rm"(i) : "r"(a), "r"(b), "r"(c), "rm"(d), "rm"(e), +# "rm"(f), "0"((int)l) : CLOBBERS); +name: tied_subreg_input +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-LABEL: name: tied_subreg_input + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.3) + ; CHECK-NEXT: renamable $rcx = COPY $r8 + ; CHECK-NEXT: $r8 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3) + ; CHECK-NEXT: renamable $rax = COPY killed $r9 + ; CHECK-NEXT: renamable $r9 = MOV64ri 42 + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, killed renamable $rcx :: (store (s64) into %stack.0) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, killed renamable $rax :: (store (s64) into %stack.1) + ; CHECK-NEXT: $ecx = COPY $edi + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR32 foldable, def renamable $ecx, reguse:GR64, killed renamable $rsi, reguse:GR64, killed renamable $rdx, reguse:GR64, killed renamable $r8, mem:m, %stack.0, 1, $noreg, 0, $noreg, mem:m, %stack.1, 1, $noreg, 0, $noreg, reguse:GR64 foldable, killed renamable $r9, reguse tiedto:$0, killed renamable $ecx(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0), (load (s64) from %stack.1) + ; CHECK-NEXT: MOV32mr %stack.2, 1, $noreg, 0, $noreg, $ecx :: (store (s32) into %stack.2) + ; CHECK-NEXT: $eax = MOV32rm %stack.2, 1, $noreg, 0, $noreg :: (load (s32) from %stack.2) + ; CHECK-NEXT: RET 0, killed $eax + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + %6:gr64 = MOV64ri 42 + INLINEASM &"", sideeffect attdialect, regdef:GR32 foldable, def %7:gr32, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64 foldable, %4, reguse:GR64 foldable, %5, reguse:GR64 foldable, %6, reguse tiedto:$0, %0.sub_32bit(tied-def 3), clobber, implicit-def early-clobber $rax, clobber, implicit-def early-clobber $rbx, clobber, implicit-def early-clobber $rbp, clobber, implicit-def early-clobber $r10, clobber, implicit-def early-clobber $r11, clobber, implicit-def early-clobber $r12, clobber, implicit-def early-clobber $r13, clobber, implicit-def early-clobber $r14, clobber, implicit-def early-clobber $r15 + $eax = COPY %7 + RET 0, $eax +... _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
