https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/237041
Split 64-bit relocations into the 32-bit halves when applying a subregister extract, similar to how immediates are modified when folding. The same split is applied when 64-bit mov pseudos are expanded. Co-authored-by: Claude Opus 5.5 <[email protected]> >From 12be4828ea10d37edef3a68c335558427b2b95ae Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Thu, 22 Oct 2026 09:04:43 -0700 Subject: [PATCH] AMDGPU: Fold global address into subregister use with lo/hi relocation Split 64-bit relocations into the 32-bit halves when applying a subregister extract, similar to how immediates are modified when folding. The same split is applied when 64-bit mov pseudos are expanded. Co-authored-by: Claude Opus 5.5 <[email protected]> --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 106 +++++++++++------- llvm/lib/Target/AMDGPU/SIInstrInfo.h | 9 ++ .../Target/AMDGPU/SIShrinkInstructions.cpp | 23 ++-- .../shrink-instructions-fold-imm-subreg.mir | 13 +-- 4 files changed, 92 insertions(+), 59 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 43ee5330e770cb..81e27e61a9661b 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -133,47 +133,33 @@ static bool canRemat(const MachineInstr &MI) { return false; } -// Split relocation flags for 64-bit global-address materialization into a -// common base and the hi/lo relocation variants. -static std::tuple<unsigned, unsigned, unsigned> -splitGlobalAddressRelocFlags(const GCNSubtarget &ST, - const MachineOperand &SrcOp) { - const unsigned BaseFlags = SrcOp.getTargetFlags() & ~SIInstrInfo::MO_MASK; - const unsigned Reloc = SrcOp.getTargetFlags() & SIInstrInfo::MO_MASK; +// Split relocation flags for 64-bit global-address materialization into the +// flags for the lo and hi halves. +static std::pair<unsigned, unsigned> +splitGlobalAddressRelocFlags(const MachineOperand &SrcOp) { + const unsigned Flags = SrcOp.getTargetFlags(); // Infer the relocation type from the existing flags on the global operand. // The relocation type should have been determined earlier in the pipeline. - unsigned LoReloc, HiReloc; - switch (Reloc) { - case SIInstrInfo::MO_REL32_LO: - case SIInstrInfo::MO_REL32_HI: - case SIInstrInfo::MO_REL64: - LoReloc = SIInstrInfo::MO_REL32_LO; - HiReloc = SIInstrInfo::MO_REL32_HI; - break; - case SIInstrInfo::MO_GOTPCREL32_LO: - case SIInstrInfo::MO_GOTPCREL32_HI: - LoReloc = SIInstrInfo::MO_GOTPCREL32_LO; - HiReloc = SIInstrInfo::MO_GOTPCREL32_HI; - break; + switch (Flags & SIInstrInfo::MO_MASK) { case SIInstrInfo::MO_GOTPCREL: - case SIInstrInfo::MO_GOTPCREL64: + case SIInstrInfo::MO_GOTPCREL64: { // For 64-bit GOT-relative, use the 64-bit relocation. - LoReloc = SIInstrInfo::MO_GOTPCREL64; - HiReloc = SIInstrInfo::MO_GOTPCREL64; - break; - case SIInstrInfo::MO_ABS32_LO: - case SIInstrInfo::MO_ABS32_HI: - case SIInstrInfo::MO_ABS64: - LoReloc = SIInstrInfo::MO_ABS32_LO; - HiReloc = SIInstrInfo::MO_ABS32_HI; - break; + const unsigned GotFlags = + (Flags & ~SIInstrInfo::MO_MASK) | SIInstrInfo::MO_GOTPCREL64; + return {GotFlags, GotFlags}; + } default: - llvm_unreachable("unknown relocation type for global address"); break; } - return {BaseFlags, LoReloc, HiReloc}; + std::optional<unsigned> LoFlags = + SIInstrInfo::extractSubregFromGlobalAddressFlags(Flags, AMDGPU::sub0); + std::optional<unsigned> HiFlags = + SIInstrInfo::extractSubregFromGlobalAddressFlags(Flags, AMDGPU::sub1); + if (!LoFlags || !HiFlags) + llvm_unreachable("unknown relocation type for global address"); + return {*LoFlags, *HiFlags}; } bool SIInstrInfo::isReMaterializableImpl( @@ -2261,14 +2247,12 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { // shortcut cannot apply. const GlobalValue *GV = SrcOp.getGlobal(); int64_t Offset = SrcOp.getOffset(); - unsigned BaseFlags, LoReloc, HiReloc; - std::tie(BaseFlags, LoReloc, HiReloc) = - splitGlobalAddressRelocFlags(ST, SrcOp); + auto [LoFlags, HiFlags] = splitGlobalAddressRelocFlags(SrcOp); BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) - .addGlobalAddress(GV, Offset, BaseFlags | LoReloc); + .addGlobalAddress(GV, Offset, LoFlags); BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) - .addGlobalAddress(GV, Offset, BaseFlags | HiReloc); + .addGlobalAddress(GV, Offset, HiFlags); } else if (SrcOp.isImm()) { APInt Imm(64, SrcOp.getImm()); APInt Lo(32, Imm.getLoBits(32).getZExtValue()); @@ -2337,14 +2321,12 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); const GlobalValue *GV = SrcOp.getGlobal(); int64_t Offset = SrcOp.getOffset(); - unsigned BaseFlags, LoReloc, HiReloc; - std::tie(BaseFlags, LoReloc, HiReloc) = - splitGlobalAddressRelocFlags(ST, SrcOp); + auto [LoFlags, HiFlags] = splitGlobalAddressRelocFlags(SrcOp); BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) - .addGlobalAddress(GV, Offset, BaseFlags | LoReloc); + .addGlobalAddress(GV, Offset, LoFlags); BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) - .addGlobalAddress(GV, Offset, BaseFlags | HiReloc); + .addGlobalAddress(GV, Offset, HiFlags); MI.eraseFromParent(); break; } @@ -3835,6 +3817,46 @@ std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm, llvm_unreachable("covered subregister switch"); } +std::optional<unsigned> +SIInstrInfo::extractSubregFromGlobalAddressFlags(unsigned TargetFlags, + unsigned SubRegIndex) { + if (SubRegIndex == AMDGPU::NoSubRegister) + return TargetFlags; + + unsigned LoReloc, HiReloc; + switch (TargetFlags & MO_MASK) { + case MO_REL32_LO: + case MO_REL32_HI: + case MO_REL64: + LoReloc = MO_REL32_LO; + HiReloc = MO_REL32_HI; + break; + case MO_GOTPCREL32_LO: + case MO_GOTPCREL32_HI: + LoReloc = MO_GOTPCREL32_LO; + HiReloc = MO_GOTPCREL32_HI; + break; + case MO_ABS32_LO: + case MO_ABS32_HI: + case MO_ABS64: + LoReloc = MO_ABS32_LO; + HiReloc = MO_ABS32_HI; + break; + default: + return std::nullopt; + } + + const unsigned BaseFlags = TargetFlags & ~MO_MASK; + switch (SubRegIndex) { + case AMDGPU::sub0: + return BaseFlags | LoReloc; + case AMDGPU::sub1: + return BaseFlags | HiReloc; + default: + return std::nullopt; + } +} + static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) { switch (Opc) { case AMDGPU::V_MAC_F16_e32: diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 72e9cda785036a..7bfd54869fcaf4 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -495,6 +495,15 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal, unsigned SubRegIndex); + /// Return the target flags for a global address operand reading subregister + /// \p SubRegIndex of a 64-bit global address with \p TargetFlags. + /// e.g. %addr = S_MOV_B64 target-flags(amdgpu-abs64) @gv + /// USE %addr.sub1 + /// This will return amdgpu-abs32-hi + static std::optional<unsigned> + extractSubregFromGlobalAddressFlags(unsigned TargetFlags, + unsigned SubRegIndex); + bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const final; diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp index f89c7534640326..22c62a156506da 100644 --- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp +++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp @@ -119,16 +119,21 @@ bool SIShrinkInstructions::foldImmediates(MachineInstr &MI, ConstantFolded = true; } } - } else if (!Src0.getSubReg() && - TII->isOperandLegal(MI, Src0Idx, &MovSrc)) { - if (MovSrc.isFI()) { - Src0.ChangeToFrameIndex(MovSrc.getIndex()); - ConstantFolded = true; - } else if (MovSrc.isGlobal()) { - Src0.ChangeToGA(MovSrc.getGlobal(), MovSrc.getOffset(), - MovSrc.getTargetFlags()); - ConstantFolded = true; + } else if (MovSrc.isGlobal()) { + if (std::optional<unsigned> Flags = + SIInstrInfo::extractSubregFromGlobalAddressFlags( + MovSrc.getTargetFlags(), Src0.getSubReg())) { + MachineOperand GAOp = MachineOperand::CreateGA( + MovSrc.getGlobal(), MovSrc.getOffset(), *Flags); + if (TII->isOperandLegal(MI, Src0Idx, &GAOp)) { + Src0.ChangeToGA(MovSrc.getGlobal(), MovSrc.getOffset(), *Flags); + ConstantFolded = true; + } } + } else if (MovSrc.isFI() && !Src0.getSubReg() && + TII->isOperandLegal(MI, Src0Idx, &MovSrc)) { + Src0.ChangeToFrameIndex(MovSrc.getIndex()); + ConstantFolded = true; } if (ConstantFolded) { diff --git a/llvm/test/CodeGen/AMDGPU/shrink-instructions-fold-imm-subreg.mir b/llvm/test/CodeGen/AMDGPU/shrink-instructions-fold-imm-subreg.mir index a1b37062fc32b1..84d2a66ec592d1 100644 --- a/llvm/test/CodeGen/AMDGPU/shrink-instructions-fold-imm-subreg.mir +++ b/llvm/test/CodeGen/AMDGPU/shrink-instructions-fold-imm-subreg.mir @@ -2,8 +2,8 @@ # RUN: llc -mtriple=amdgpu9.00 -run-pass=si-shrink-instructions -o - %s | FileCheck %s # Folding a 64-bit move immediate into a subregister use must only fold -# the immediate value of the used half. A global address cannot be folded -# into a subregister use. +# the immediate value of the used half. A global address should use the +# relocation for the used half. --- name: fold_v_mov_b64_sub0 @@ -95,8 +95,7 @@ body: | ; CHECK: liveins: $vgpr0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; CHECK-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 target-flags(amdgpu-abs64) @s_mov_b64_global_abs64_sub0 + 8 - ; CHECK-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 [[S_MOV_B64_]].sub0, [[COPY]], implicit $exec + ; CHECK-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 target-flags(amdgpu-abs32-lo) @s_mov_b64_global_abs64_sub0 + 8, [[COPY]], implicit $exec ; CHECK-NEXT: $vgpr0 = COPY [[V_AND_B32_e32_]] ; CHECK-NEXT: SI_RETURN implicit $vgpr0 %0:vgpr_32 = COPY $vgpr0 @@ -116,8 +115,7 @@ body: | ; CHECK: liveins: $vgpr0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; CHECK-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 target-flags(amdgpu-abs64) @s_mov_b64_global_abs64_sub1 + 8 - ; CHECK-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 [[S_MOV_B64_]].sub1, [[COPY]], implicit $exec + ; CHECK-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 target-flags(amdgpu-abs32-hi) @s_mov_b64_global_abs64_sub1 + 8, [[COPY]], implicit $exec ; CHECK-NEXT: $vgpr0 = COPY [[V_AND_B32_e32_]] ; CHECK-NEXT: SI_RETURN implicit $vgpr0 %0:vgpr_32 = COPY $vgpr0 @@ -137,8 +135,7 @@ body: | ; CHECK: liveins: $vgpr0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; CHECK-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 target-flags(amdgpu-rel64) @s_mov_b64_global_rel64_sub1 + 8 - ; CHECK-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 [[S_MOV_B64_]].sub1, [[COPY]], implicit $exec + ; CHECK-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 target-flags(amdgpu-rel32-hi) @s_mov_b64_global_rel64_sub1 + 8, [[COPY]], implicit $exec ; CHECK-NEXT: $vgpr0 = COPY [[V_AND_B32_e32_]] ; CHECK-NEXT: SI_RETURN implicit $vgpr0 %0:vgpr_32 = COPY $vgpr0 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
