https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/229627
>From 674292485d0784dca0aed5949193405831eb8cbd Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Thu, 8 Oct 2026 00:28:24 -0700 Subject: [PATCH 1/7] [TargetInstrInfo] Fix folding inline asm operands next to tied operands foldInlineAsmMemOperand() swapped a register operand for the target's memory operands with MachineInstr::removeOperand(), which asserts when a later operand is tied, because moving it would break the tie. Inline asm lists every input after every output, so folding any operand that comes before another tied pair asserted, e.g. an "rm" input followed by the input of a "+r" operand, or one of two "+rm" operands. Without assertions, the moved operands kept stale tie indices. Untie the operands, rebuild the operand list, and re-tie the remaining pairs at their new positions. It also gave up when the register appears in more than one operand, e.g. one value passed to two "rm" operands, which left the greedy allocator unable to spill that value at all. Fold every such operand into the stack slot. Finally, take MayLoad from the folded operands rather than from every read of the register: a folded def whose tied use is another virtual register (as when the fast register allocator lowers tied operands itself) still reads the slot, and an unrelated register read of the same value doesn't. Nothing in tree marks an inline asm register operand foldable yet, so the test sets the "foldable" flag in MIR and runs the greedy allocator. Assisted-by: Claude Opus 5.5 Co-authored-by: Nick Desaulniers <[email protected]> --- llvm/lib/CodeGen/TargetInstrInfo.cpp | 117 +++++++++++++----- .../CodeGen/X86/greedy-inline-asm-fold.mir | 108 ++++++++++++++++ 2 files changed, 193 insertions(+), 32 deletions(-) create mode 100644 llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index ae64c0086aa017..aeec7568e7d77a 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -650,29 +650,66 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI, return NewMI; } -static void foldInlineAsmMemOperand(MachineInstr *MI, unsigned OpNo, int FI, - const TargetInstrInfo &TII) { - // If the machine operand is tied, untie it first. - if (MI->getOperand(OpNo).isTied()) { - unsigned TiedTo = MI->findTiedOperandIdx(OpNo); - MI->untieRegOperand(OpNo); - // Intentional recursion! - foldInlineAsmMemOperand(MI, TiedTo, FI, TII); +/// Rewrite the register operands \p Ops of the inline asm \p MI to refer to +/// stack slot \p FI. The use tied to a folded def is folded along with it, +/// since the two name a single location. +static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops, + int FI, const TargetInstrInfo &TII) { + SmallVector<MachineOperand, 5> MemOps; + TII.getFrameIndexOperands(MemOps, FI); + assert(!MemOps.empty() && "getFrameIndexOperands didn't create any operands"); + InlineAsm::Flag MemFlag(InlineAsm::Kind::Mem, MemOps.size()); + MemFlag.setMemConstraint(InlineAsm::ConstraintCode::m); + + // Find the uses to fold along with their defs, and the ties to keep. A + // folded operand becomes MemOps.size() operands, moving every later one, and + // MachineInstr can't move a tied operand. So untie everything (untying a use + // also unties its def), re-add the operands from the first folded one on, + // and re-tie the pairs that are still registers at their new positions. + SmallVector<unsigned, 4> FoldOps(Ops); + SmallVector<std::pair<unsigned, unsigned>, 4> Ties; + for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); + I != E; ++I) { + unsigned DefIdx; + if (!MI.isRegTiedToDefOperand(I, &DefIdx)) + continue; + + MI.untieRegOperand(I); + if (is_contained(Ops, DefIdx)) + FoldOps.push_back(I); + else + Ties.emplace_back(DefIdx, I); } - SmallVector<MachineOperand, 5> NewOps; - TII.getFrameIndexOperands(NewOps, FI); - assert(!NewOps.empty() && "getFrameIndexOperands didn't create any operands"); - MI->removeOperand(OpNo); - MI->insert(MI->operands_begin() + OpNo, NewOps); + unsigned First = *llvm::min_element(FoldOps); + SmallVector<MachineOperand, 16> Tail(MI.operands_begin() + First, + MI.operands_end()); + while (MI.getNumOperands() > First) + MI.removeOperand(MI.getNumOperands() - 1); + + SmallVector<unsigned, 16> NewIdx; + for (auto [I, MO] : enumerate(Tail)) { + NewIdx.push_back(MI.getNumOperands()); + if (!is_contained(FoldOps, First + I)) { + MI.addOperand(MO); + continue; + } + + // Only the first register after a flag can be folded, so the operand just + // before this one is its flag. It now describes the memory operand. + MachineOperand &FlagMO = MI.getOperand(MI.getNumOperands() - 1); + assert(InlineAsm::Flag(FlagMO.getImm()).getNumOperandRegisters() == 1 && + "cannot fold one register of a multi-register operand"); + FlagMO.setImm(MemFlag); + for (const MachineOperand &MemOp : MemOps) + MI.addOperand(MemOp); + } - // Change the previous operand to a MemKind InlineAsm::Flag. The second param - // is the per-target number of operands that represent the memory operand - // excluding this one (MD). This includes MO. - InlineAsm::Flag F(InlineAsm::Kind::Mem, NewOps.size()); - F.setMemConstraint(InlineAsm::ConstraintCode::m); - MachineOperand &MD = MI->getOperand(OpNo - 1); - MD.setImm(F); + auto Remap = [&](unsigned Idx) { + return Idx < First ? Idx : NewIdx[Idx - First]; + }; + for (auto [DefIdx, UseIdx] : Ties) + MI.tieOperands(Remap(DefIdx), Remap(UseIdx)); } // Returns nullptr if not possible to fold. @@ -680,29 +717,45 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, int FI, const TargetInstrInfo &TII) { assert(MI.isInlineAsm() && "wrong opcode"); - if (Ops.size() > 1) - return nullptr; - unsigned Op = Ops[0]; - assert(Op && "should never be first operand"); - assert(MI.getOperand(Op).isReg() && "shouldn't be folding non-reg operands"); - if (!MI.mayFoldInlineAsmRegOp(Op)) + // Sanity check that there are operands to fold. + if (Ops.empty()) return nullptr; - MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI); + // Every operand in Ops holds the same register (e.g. one value passed to two + // "rm" operands), so they all fold to the same stack slot. The asm reads + // the slot through a use, and through a def tied to a use, which need not + // be the same virtual register when the allocator lowers ties itself. + bool Reads = false; + bool Writes = false; + for (unsigned Op : Ops) { + assert(Op && "should never be first operand"); + const MachineOperand &MO = MI.getOperand(Op); + assert(MO.isReg() && "shouldn't be folding non-reg operands"); + + // A tied use is only ever folded along with its def. + if (MI.isRegTiedToDefOperand(Op) || !MI.mayFoldInlineAsmRegOp(Op)) + return nullptr; - foldInlineAsmMemOperand(&NewMI, Op, FI, TII); + Reads |= MO.readsReg(); + if (MO.isDef()) { + Writes = true; + Reads |= + MO.isTied() && MI.getOperand(MI.findTiedOperandIdx(Op)).readsReg(); + } + } + + MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI); + foldInlineAsmMemOperands(NewMI, Ops, FI, TII); // Update mayload/maystore metadata, and memoperands. - const VirtRegInfo &RI = - AnalyzeVirtRegInBundle(MI, MI.getOperand(Op).getReg()); MachineOperand &ExtraMO = NewMI.getOperand(InlineAsm::MIOp_ExtraInfo); MachineMemOperand::Flags Flags = MachineMemOperand::MONone; - if (RI.Reads) { + if (Reads) { ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayLoad); Flags |= MachineMemOperand::MOLoad; } - if (RI.Writes) { + if (Writes) { ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore); Flags |= MachineMemOperand::MOStore; } diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir new file mode 100644 index 00000000000000..f7e1c695ebfc48 --- /dev/null +++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir @@ -0,0 +1,108 @@ +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy \ +# RUN: -verify-machineinstrs %s -o - | FileCheck %s + +# The greedy register allocator spills a value used by a foldable ("rm") +# inline asm operand by folding the operand to the value's stack slot. The +# clobbers below leave six GPRs, and the asm needs seven registers, so one +# value is spilled. Folding replaces an operand with the five operands of a +# memory reference, moving every operand after it, which must keep any tied +# operands tied. +# +# Each test is derived from the C function and inline asm statement in its +# comment, where CLOBBERS is: +# +# "rax", "rbx", "rbp", "r10", "r11", "r12", "r13", "r14", "r15" + +--- +# One value passed to two "rm" operands: both operands fold to the same stack +# slot. +# +# void rm_same_value_twice(long a, long b, long c, long d, long e, long f, +# long g) { +# asm volatile("" : : "rm"(a), "rm"(a), "r"(b), "r"(c), "r"(d), "r"(e), +# "r"(f), "r"(g) : CLOBBERS); +# } +name: rm_same_value_twice +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: rm_same_value_twice + ; CHECK: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, reguse:GR64, + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64 foldable, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + RET 0 +... +--- +# Folding the "rm" input moves the input tied to the "=r" output after it. +# +# long rm_before_tied(long a, long b, long c, long d, long e, long f, +# long g) { +# asm volatile("" : "+r"(g) : "rm"(a), "r"(b), "r"(c), "r"(d), "r"(e), +# "r"(f) : CLOBBERS); +# return g; +# } +name: rm_before_tied +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: rm_before_tied + ; CHECK: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def %{{[0-9]+}}, mem:m, %stack.{{[0-9]+}}, 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), clobber, + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + INLINEASM &"", sideeffect attdialect, regdef:GR64, def %6, reguse:GR64 foldable, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse tiedto:$0, %6(tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + $rax = COPY %6 + RET 0, $rax +... +--- +# Two read-write "+rm" operands: folding one tied pair, which reads and writes +# the stack slot, keeps the other pair tied. +# +# struct pair { long a, b; }; +# +# struct pair two_tied_rm(long a, long b, long c, long d, long e, long f, +# long g) { +# asm volatile("" : "+rm"(a), "+rm"(b) : "r"(c), "r"(d), "r"(e), "r"(f), +# "r"(g) : CLOBBERS); +# return (struct pair){a, b}; +# } +name: two_tied_rm +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + + ; CHECK-LABEL: name: two_tied_rm + ; CHECK: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def %{{[0-9]+}}, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, clobber, + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %0, regdef:GR64 foldable, def %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, reguse tiedto:$0, %0(tied-def 3), reguse tiedto:$1, %1(tied-def 5), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + $rax = COPY %0 + $rdx = COPY %1 + RET 0, $rax, $rdx +... >From 7c4a3509ebb310feefcaa7be28a577d9677c2c20 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Thu, 15 Oct 2026 17:55:29 -0700 Subject: [PATCH 2/7] [TargetInstrInfo] Address review: autogenerate greedy-inline-asm-fold.mir checks Generate the checks with update_mir_test_checks.py so that the spills and reloads around each folded inline asm are verified, not just the INLINEASM itself. Fix the "=r" comment to describe the C "+r" operand, note that which "+rm" pair greedy folds is its own choice, and add a "+&rm" case on an uninitialized value showing that folding a pair whose tied use is undef stores to the slot without loading from it. Assisted-by: Claude Opus 5.5 --- .../CodeGen/X86/greedy-inline-asm-fold.mir | 102 ++++++++++++++++-- 1 file changed, 94 insertions(+), 8 deletions(-) diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir index f7e1c695ebfc48..1dcc116f8ec11d 100644 --- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir +++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir @@ -1,3 +1,4 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy \ # RUN: -verify-machineinstrs %s -o - | FileCheck %s @@ -29,9 +30,18 @@ fixedStack: body: | bb.0: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 - ; CHECK-LABEL: name: rm_same_value_twice - ; CHECK: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, reguse:GR64, + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdx + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rcx + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9 + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0 %0:gr64 = COPY $rdi %1:gr64 = COPY $rsi %2:gr64 = COPY $rdx @@ -43,7 +53,8 @@ body: | RET 0 ... --- -# Folding the "rm" input moves the input tied to the "=r" output after it. +# Folding the "rm" input moves the input that "+r"(g) ties to its output, +# which comes after the "rm" input in the operand list. # # long rm_before_tied(long a, long b, long c, long d, long e, long f, # long g) { @@ -58,9 +69,19 @@ fixedStack: body: | bb.0: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 - ; CHECK-LABEL: name: rm_before_tied - ; CHECK: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def %{{[0-9]+}}, mem:m, %stack.{{[0-9]+}}, 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), clobber, + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdx + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rcx + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9 + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def [[MOV64rm]], mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse tiedto:$0, [[MOV64rm]](tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: $rax = COPY [[MOV64rm]] + ; CHECK-NEXT: RET 0, $rax %0:gr64 = COPY $rdi %1:gr64 = COPY $rsi %2:gr64 = COPY $rdx @@ -74,7 +95,9 @@ body: | ... --- # Two read-write "+rm" operands: folding one tied pair, which reads and writes -# the stack slot, keeps the other pair tied. +# the stack slot, keeps the other pair tied. Which pair is folded is up to the +# greedy allocator's spill weights, not the asm statement; it currently folds +# "+rm"(b), so both its output and its tied input become "m" on the same slot. # # struct pair { long a, b; }; # @@ -91,9 +114,25 @@ fixedStack: body: | bb.0: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 - ; CHECK-LABEL: name: two_tied_rm - ; CHECK: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def %{{[0-9]+}}, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, clobber, + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.4) + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.0) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1) + ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.2) + ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $r9 :: (store (s64) into %stack.3) + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = lr-split COPY [[COPY]] + ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2) + ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4) + ; CHECK-NEXT: $rax = COPY [[COPY1]] + ; CHECK-NEXT: $rdx = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4) + ; CHECK-NEXT: RET 0, $rax, $rdx %0:gr64 = COPY $rdi %1:gr64 = COPY $rsi %2:gr64 = COPY $rdx @@ -106,3 +145,50 @@ body: | $rdx = COPY %1 RET 0, $rax, $rdx ... +--- +# A read-write "+rm" operand on an uninitialized value: the tied input is +# undef, so folding the pair writes the stack slot but doesn't read it. The +# folded asm is "maystore" without "mayload" and has only a store memoperand. +# The output is early-clobber so that it can't share a register with one of +# the inputs; otherwise six GPRs would be enough and nothing would be folded. +# +# long tied_rm_undef(long b, long c, long d, long e, long f, long g) { +# long a; +# asm volatile("" : "+&rm"(a) : "r"(b), "r"(c), "r"(d), "r"(e), "r"(f), +# "r"(g) : CLOBBERS); +# return a; +# } +name: tied_rm_undef +tracksRegLiveness: true +registers: + - { id: 0, class: gr64 } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-LABEL: name: tied_rm_undef + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1) + ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.2) + ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.3) + ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.4) + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $r9 + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2) + ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3) + ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4) + ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5) + ; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5) + ; CHECK-NEXT: RET 0, $rax + %1:gr64 = COPY $rdi + %2:gr64 = COPY $rsi + %3:gr64 = COPY $rdx + %4:gr64 = COPY $rcx + %5:gr64 = COPY $r8 + %6:gr64 = COPY $r9 + INLINEASM &"", sideeffect attdialect, regdef-ec:GR64 foldable, def early-clobber %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, reguse tiedto:$0, undef %0(tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + $rax = COPY %0 + RET 0, $rax +... >From 7d7f2cdb5f61c501662d44dfa101bc4cb415171b Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Thu, 22 Oct 2026 01:53:58 -0700 Subject: [PATCH 3/7] [TargetInstrInfo] Address review: don't fold one register of a multi-register operand A memory operand replaces exactly one register, so return nullptr for a folded operand whose flag holds more than one, rather than relying on an assert that release builds drop. Also assert that every folded operand holds the same register, and say that a caller folding a def whose tied use is another register must store that register's value into the slot. Test both the new bail-out and folding a rematerialized load into two "rm" operands, which takes the fold-a-load path. Assisted-by: Claude Opus 5.5 --- llvm/lib/CodeGen/TargetInstrInfo.cpp | 12 +++- .../CodeGen/X86/greedy-inline-asm-fold.mir | 72 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index aeec7568e7d77a..30f64a5d8c6b5c 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -725,18 +725,28 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI, // Every operand in Ops holds the same register (e.g. one value passed to two // "rm" operands), so they all fold to the same stack slot. The asm reads // the slot through a use, and through a def tied to a use, which need not - // be the same virtual register when the allocator lowers ties itself. + // be the same virtual register when the allocator lowers ties itself. In + // that case the caller must store the tied use's value into the slot before + // the asm, as RegAllocFast does. bool Reads = false; bool Writes = false; for (unsigned Op : Ops) { assert(Op && "should never be first operand"); const MachineOperand &MO = MI.getOperand(Op); assert(MO.isReg() && "shouldn't be folding non-reg operands"); + assert(MO.getReg() == MI.getOperand(Ops[0]).getReg() && + "folded operands must hold the same register"); // A tied use is only ever folded along with its def. if (MI.isRegTiedToDefOperand(Op) || !MI.mayFoldInlineAsmRegOp(Op)) return nullptr; + // A memory operand replaces exactly one register, so one register of a + // multi-register operand (e.g. an i128 in two GR64s) can't be folded. + if (InlineAsm::Flag(MI.getOperand(Op - 1).getImm()) + .getNumOperandRegisters() != 1) + return nullptr; + Reads |= MO.readsReg(); if (MO.isDef()) { Writes = true; diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir index 1dcc116f8ec11d..a89b9791e6f93e 100644 --- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir +++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir @@ -192,3 +192,75 @@ body: | $rax = COPY %0 RET 0, $rax ... +--- +# One register of a two-register operand can't be folded, since a memory +# operand replaces exactly one register. The first asm clobbers every GPR, so +# the value is spilled across it; the second asm then gets a reload instead of +# a fold. The flag is written as an immediate, since a named flag always has +# one register: it is "reguse:GR64 foldable" for two registers. Nothing in +# tree marks such an operand foldable (an i128 "rm" operand picks memory), so +# this only guards the fold. +# +# void rm_two_registers(__int128 a) { +# asm volatile("" : : : ALL_GPRS); +# asm volatile("" : : "rm"(a)); +# } +name: rm_two_registers +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi, $rsi + ; CHECK-LABEL: name: rm_two_registers + ; CHECK: liveins: $rdi, $rsi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm]], [[MOV64rm1]] + ; CHECK-NEXT: RET 0 + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + INLINEASM &"", sideeffect attdialect, 1078525969, %0, %1 + RET 0 +... +--- +# A value loaded from an immutable stack slot is rematerialized rather than +# spilled, by folding the load into its users. When it's passed to two "rm" +# operands, both fold to the load's stack slot. +# +# void rm_load_twice(long a, long b, long c, long d, long e, long f, +# long g) { +# asm volatile("" : : "rm"(g), "rm"(g), "r"(a), "r"(b), "r"(c), "r"(d), +# "r"(e), "r"(f) : CLOBBERS); +# } +name: rm_load_twice +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +body: | + bb.0: + liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-LABEL: name: rm_load_twice + ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8 + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9 + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: RET 0 + %0:gr64 = COPY $rdi + %1:gr64 = COPY $rsi + %2:gr64 = COPY $rdx + %3:gr64 = COPY $rcx + %4:gr64 = COPY $r8 + %5:gr64 = COPY $r9 + %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) + INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64 foldable, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + RET 0 +... >From d232ab817519c142cd1724f793f2a62a492fc4b3 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Fri, 23 Oct 2026 18:35:04 -0700 Subject: [PATCH 4/7] Update llvm/lib/CodeGen/TargetInstrInfo.cpp Co-authored-by: Nick Desaulniers <[email protected]> --- llvm/lib/CodeGen/TargetInstrInfo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index 30f64a5d8c6b5c..463a887e3be87b 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -727,7 +727,7 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI, // the slot through a use, and through a def tied to a use, which need not // be the same virtual register when the allocator lowers ties itself. In // that case the caller must store the tied use's value into the slot before - // the asm, as RegAllocFast does. + // the asm. bool Reads = false; bool Writes = false; for (unsigned Op : Ops) { >From b0e54933125140d0f234176063d9efc68f8d5cd1 Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Fri, 23 Oct 2026 18:55:17 -0700 Subject: [PATCH 5/7] Move folding register checks into mayFoldInlineAsmRegOp so that it's available everywhere. --- llvm/lib/CodeGen/MachineInstr.cpp | 9 +++++++++ llvm/lib/CodeGen/TargetInstrInfo.cpp | 8 +------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index 4978967c7cb112..32963d1e357b3a 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -2786,13 +2786,22 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const { if (!getOperand(OpId).isReg()) return false; + if (isRegTiedToDefOperand(OpId)) + return false; + const MachineOperand &MD = getOperand(OpId - 1); if (!MD.isImm()) return false; + // A memory operand replaces exactly one register, so one register of a + // multi-register operand (e.g. an i128 in two GR64s) can't be folded. InlineAsm::Flag F(MD.getImm()); + if (F.getNumOperandRegisters() != 1) + return false; + if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind()) return F.getRegMayBeFolded(); + return false; } diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index 463a887e3be87b..4418e3fc7a8248 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -738,13 +738,7 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI, "folded operands must hold the same register"); // A tied use is only ever folded along with its def. - if (MI.isRegTiedToDefOperand(Op) || !MI.mayFoldInlineAsmRegOp(Op)) - return nullptr; - - // A memory operand replaces exactly one register, so one register of a - // multi-register operand (e.g. an i128 in two GR64s) can't be folded. - if (InlineAsm::Flag(MI.getOperand(Op - 1).getImm()) - .getNumOperandRegisters() != 1) + if (!MI.mayFoldInlineAsmRegOp(Op)) return nullptr; Reads |= MO.readsReg(); >From 9e860ba4cd2fa7fc247851653b40aeb23de0672c Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Fri, 23 Oct 2026 19:06:09 -0700 Subject: [PATCH 6/7] [TargetInstrInfo] Address review: don't fold a subregister that isn't at the start of the slot A folded operand refers to the start of the stack slot, which holds the whole register, so a subregister is only there if it's at bit offset 0 on a little-endian target. Folding x86's sub_8bit_hi read byte 0 instead of byte 1. Folding several operands at once made this worse: two operands holding different subregisters of one value (e.g. "Qm"(lo), "Qm"(hi)) both read byte 0. Check the subregister in mayFoldInlineAsmRegOp, so the spill weights agree with what folds. A tied def is covered too, since its tied use has the same subregister. Assisted-by: Claude Opus 5.5 --- llvm/include/llvm/CodeGen/MachineInstr.h | 3 +- llvm/lib/CodeGen/MachineInstr.cpp | 11 ++ .../CodeGen/X86/greedy-inline-asm-fold.mir | 112 ++++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index f66dff6e5bb3b1..731b92f555f4c8 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -1440,7 +1440,8 @@ class MachineInstr } /// Returns true if the register operand can be folded with a load or store /// into a frame index. Does so by checking the InlineAsm::Flag immediate - /// operand at OpId - 1. + /// operand at OpId - 1, and that the register (or subregister) starts at the + /// beginning of the stack slot. LLVM_ABI bool mayFoldInlineAsmRegOp(unsigned OpId) const; LLVM_ABI bool isStackAligningInlineAsm() const; diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index 32963d1e357b3a..e43c5a5fa50998 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -2799,6 +2799,17 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const { if (F.getNumOperandRegisters() != 1) return false; + // A memory operand refers to the start of the stack slot, which holds the + // whole register. A subregister is found there only if it's at bit offset 0 + // on a little-endian target; e.g. x86's sub_8bit_hi is at byte 1. (The use + // tied to a def, which is folded along with it, has the def's subregister.) + if (unsigned SubReg = getOperand(OpId).getSubReg()) { + const MachineFunction &MF = *getMF(); + if (!MF.getDataLayout().isLittleEndian() || + MF.getSubtarget().getRegisterInfo()->getSubRegIdxOffset(SubReg) != 0) + return false; + } + if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind()) return F.getRegMayBeFolded(); diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir index a89b9791e6f93e..9113acd3058aa1 100644 --- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir +++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir @@ -227,6 +227,118 @@ body: | RET 0 ... --- +# A stack slot holds the whole register, so a subregister operand can be +# folded only if the subregister starts at the beginning of the slot. Each of +# these tests spills a value across an asm that clobbers every GPR, as above, +# and passes its low byte, its second byte, or both to a "Qm" operand of a +# second asm. +# +# The low byte (sub_8bit) is at the start of the slot, so it's folded. +# +# void qm_low_byte(unsigned long a) { +# asm volatile("" : : : ALL_GPRS); +# asm volatile("" : : "Qm"((unsigned char)a)); +# } +name: qm_low_byte +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi + ; CHECK-LABEL: name: qm_low_byte + ; CHECK: liveins: $rdi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: RET 0 + %0:gr64_abcd = COPY $rdi + INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, %0.sub_8bit + RET 0 +... +--- +# The second byte (sub_8bit_hi) is at byte 1 of the slot, so it gets a reload +# instead of a fold to the start of the slot. +# +# void qm_high_byte(unsigned long a) { +# asm volatile("" : : : ALL_GPRS); +# asm volatile("" : : "Qm"((unsigned char)(a >> 8))); +# } +name: qm_high_byte +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi + ; CHECK-LABEL: name: qm_high_byte + ; CHECK: liveins: $rdi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi + ; CHECK-NEXT: RET 0 + %0:gr64_abcd = COPY $rdi + INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_H foldable, %0.sub_8bit_hi + RET 0 +... +--- +# Both bytes are subregisters of one value. Folding both operands would read +# byte 0 for each, so neither is folded. +# +# void qm_both_bytes(unsigned long a) { +# asm volatile("" : : : ALL_GPRS); +# asm volatile("" : : "Qm"((unsigned char)a), +# "Qm"((unsigned char)(a >> 8))); +# } +name: qm_both_bytes +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi + ; CHECK-LABEL: name: qm_both_bytes + ; CHECK: liveins: $rdi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, [[MOV64rm]].sub_8bit, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi + ; CHECK-NEXT: RET 0 + %0:gr64_abcd = COPY $rdi + INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, %0.sub_8bit, reguse:GR8_ABCD_H foldable, %0.sub_8bit_hi + RET 0 +... +--- +# A def is folded along with the use tied to it, so neither is folded when the +# tied use is the second byte. +# +# unsigned long qm_tied_high_byte(unsigned long a) { +# asm volatile("" : : : ALL_GPRS); +# asm volatile("" : "+Qm"(((unsigned char *)&a)[1])); +# return a; +# } +name: qm_tied_high_byte +tracksRegLiveness: true +body: | + bb.0: + liveins: $rdi + ; CHECK-LABEL: name: qm_tied_high_byte + ; CHECK: liveins: $rdi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def [[MOV64rm]].sub_8bit_hi, reguse tiedto:$0, [[MOV64rm]].sub_8bit_hi(tied-def 3) + ; CHECK-NEXT: $rax = COPY [[MOV64rm]] + ; CHECK-NEXT: RET 0, $rax + %0:gr64_abcd = COPY $rdi + INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 + INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def %0.sub_8bit_hi, reguse tiedto:$0, %0.sub_8bit_hi(tied-def 3) + $rax = COPY %0 + RET 0, $rax +... +--- # A value loaded from an immutable stack slot is rematerialized rather than # spilled, by folding the load into its users. When it's passed to two "rm" # operands, both fold to the load's stack slot. >From a7a659b7f7f93e9db8a32938ab10e9849952187f Mon Sep 17 00:00:00 2001 From: Bill Wendling <[email protected]> Date: Fri, 23 Oct 2026 19:38:18 -0700 Subject: [PATCH 7/7] [TargetInstrInfo] Address review: write the two-register flag symbolically Now that the operand count of an inline asm flag can be written in MIR, use "reguse:GR64 foldable ops:2" instead of an immediate whose register class ID changes whenever a register class is added. Regenerate the checks, which now print the five operands of each x86 memory reference as "ops:5". Assisted-by: Claude Opus 5.5 --- .../CodeGen/X86/greedy-inline-asm-fold.mir | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir index 9113acd3058aa1..eceb4d4d4bc37c 100644 --- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir +++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir @@ -40,7 +40,7 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8 ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9 ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) - ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) ; CHECK-NEXT: RET 0 %0:gr64 = COPY $rdi %1:gr64 = COPY $rsi @@ -79,7 +79,7 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8 ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9 ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16) - ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def [[MOV64rm]], mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse tiedto:$0, [[MOV64rm]](tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def [[MOV64rm]], mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse tiedto:$0, [[MOV64rm]](tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0) ; CHECK-NEXT: $rax = COPY [[MOV64rm]] ; CHECK-NEXT: RET 0, $rax %0:gr64 = COPY $rdi @@ -129,7 +129,7 @@ body: | ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2) ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3) - ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4) ; CHECK-NEXT: $rax = COPY [[COPY1]] ; CHECK-NEXT: $rdx = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4) ; CHECK-NEXT: RET 0, $rax, $rdx @@ -179,7 +179,7 @@ body: | ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2) ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3) ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4) - ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5) + ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5) ; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5) ; CHECK-NEXT: RET 0, $rax %1:gr64 = COPY $rdi @@ -196,10 +196,8 @@ body: | # One register of a two-register operand can't be folded, since a memory # operand replaces exactly one register. The first asm clobbers every GPR, so # the value is spilled across it; the second asm then gets a reload instead of -# a fold. The flag is written as an immediate, since a named flag always has -# one register: it is "reguse:GR64 foldable" for two registers. Nothing in -# tree marks such an operand foldable (an i128 "rm" operand picks memory), so -# this only guards the fold. +# a fold. Nothing in tree marks such an operand foldable (an i128 "rm" operand +# picks memory), so this only guards the fold. # # void rm_two_registers(__int128 a) { # asm volatile("" : : : ALL_GPRS); @@ -218,12 +216,12 @@ body: | ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1) - ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm]], [[MOV64rm1]] + ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable ops:2, [[MOV64rm]], [[MOV64rm1]] ; CHECK-NEXT: RET 0 %0:gr64 = COPY $rdi %1:gr64 = COPY $rsi INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 - INLINEASM &"", sideeffect attdialect, 1078525969, %0, %1 + INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable ops:2, %0, %1 RET 0 ... --- @@ -249,7 +247,7 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0) ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 - ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0) ; CHECK-NEXT: RET 0 %0:gr64_abcd = COPY $rdi INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 @@ -364,7 +362,7 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8 ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9 - ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %fixed-stack.0, align 16) + ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %fixed-stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %fixed-stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %fixed-stack.0, align 16) ; CHECK-NEXT: RET 0 %0:gr64 = COPY $rdi %1:gr64 = COPY $rsi _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
