https://github.com/llvmbot updated
https://github.com/llvm/llvm-project/pull/134775
>From 73d1e8598eda19be8edd9eaaefc091228c651217 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <k...@google.com>
Date: Fri, 7 Mar 2025 00:59:25 -0800
Subject: [PATCH 1/3] [CodeGen] Avoid repeated hash lookups (NFC) (#130237)
(cherry picked from commit 616f277640129ff37d4005737a62acbd60d06ca1)
---
llvm/lib/CodeGen/ModuloSchedule.cpp | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp
b/llvm/lib/CodeGen/ModuloSchedule.cpp
index f9fe812f7e65c..8fed3a6ff709f 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -464,10 +464,12 @@ void ModuloScheduleExpander::generateExistingPhis(
InstOp1 = MRI.getVRegDef(PhiOp1);
int PhiOpStage = Schedule.getStage(InstOp1);
int StageAdj = (PhiOpStage != -1 ? PhiStage - PhiOpStage : 0);
- if (PhiOpStage != -1 && PrologStage - StageAdj >= Indirects + np &&
- VRMap[PrologStage - StageAdj - Indirects - np].count(PhiOp1)) {
- PhiOp1 = VRMap[PrologStage - StageAdj - Indirects - np][PhiOp1];
- break;
+ if (PhiOpStage != -1 && PrologStage - StageAdj >= Indirects + np) {
+ auto &M = VRMap[PrologStage - StageAdj - Indirects - np];
+ if (auto It = M.find(PhiOp1); It != M.end()) {
+ PhiOp1 = It->second;
+ break;
+ }
}
++Indirects;
}
@@ -597,8 +599,11 @@ void ModuloScheduleExpander::generateExistingPhis(
// Check if we need to rename a Phi that has been eliminated due to
// scheduling.
- if (NumStages == 0 && IsLast && VRMap[CurStageNum].count(LoopVal))
- replaceRegUsesAfterLoop(Def, VRMap[CurStageNum][LoopVal], BB, MRI, LIS);
+ if (NumStages == 0 && IsLast) {
+ auto It = VRMap[CurStageNum].find(LoopVal);
+ if (It != VRMap[CurStageNum].end())
+ replaceRegUsesAfterLoop(Def, It->second, BB, MRI, LIS);
+ }
}
}
>From d88cd35023b42c436517e98d2a0f223394337332 Mon Sep 17 00:00:00 2001
From: Hua Tian <akirat...@tencent.com>
Date: Mon, 17 Mar 2025 14:28:47 +0800
Subject: [PATCH 2/3] [llvm][CodeGen] Fix the empty interval issue in Window
Scheduler (#129204)
The interval of newly generated reg in ModuloScheduleExpander is empty.
This will cause crash at some corner case. This patch recalculate the
live intervals of these regs.
(cherry picked from commit b09b9ac1081d19c8021df8e55e96cd1325f0eed0)
---
llvm/include/llvm/CodeGen/ModuloSchedule.h | 4 +
llvm/lib/CodeGen/ModuloSchedule.cpp | 75 ++++++---
.../swp-ws-live-intervals-issue128714.mir | 157 ++++++++++++++++++
3 files changed, 215 insertions(+), 21 deletions(-)
create mode 100644
llvm/test/CodeGen/Hexagon/swp-ws-live-intervals-issue128714.mir
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h
b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index 64598ce449a44..e2fb1daf9fb4e 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -188,6 +188,9 @@ class ModuloScheduleExpander {
/// Instructions to change when emitting the final schedule.
InstrChangesTy InstrChanges;
+ /// Record the registers that need to compute live intervals.
+ SmallVector<Register> NoIntervalRegs;
+
void generatePipelinedLoop();
void generateProlog(unsigned LastStage, MachineBasicBlock *KernelBB,
ValueMapTy *VRMap, MBBVectorTy &PrologBBs);
@@ -211,6 +214,7 @@ class ModuloScheduleExpander {
void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs,
MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs,
ValueMapTy *VRMap);
+ void calculateIntervals();
bool computeDelta(MachineInstr &MI, unsigned &Delta);
void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI,
unsigned Num);
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp
b/llvm/lib/CodeGen/ModuloSchedule.cpp
index 8fed3a6ff709f..fcfb3c1f7073d 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -141,6 +141,7 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
MachineInstr *NewMI = cloneInstr(CI, MaxStageCount, StageNum);
updateInstruction(NewMI, false, MaxStageCount, StageNum, VRMap);
KernelBB->push_back(NewMI);
+ LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = CI;
}
@@ -150,6 +151,7 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
MachineInstr *NewMI = MF.CloneMachineInstr(&MI);
updateInstruction(NewMI, false, MaxStageCount, 0, VRMap);
KernelBB->push_back(NewMI);
+ LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = &MI;
}
@@ -179,6 +181,10 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
// Add branches between prolog and epilog blocks.
addBranches(*Preheader, PrologBBs, KernelBB, EpilogBBs, VRMap);
+ // The intervals of newly created virtual registers are calculated after the
+ // kernel expansion.
+ calculateIntervals();
+
delete[] VRMap;
delete[] VRMapPhi;
}
@@ -226,6 +232,7 @@ void ModuloScheduleExpander::generateProlog(unsigned
LastStage,
cloneAndChangeInstr(&*BBI, i, (unsigned)StageNum);
updateInstruction(NewMI, false, i, (unsigned)StageNum, VRMap);
NewBB->push_back(NewMI);
+ LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = &*BBI;
}
}
@@ -303,6 +310,7 @@ void ModuloScheduleExpander::generateEpilog(
MachineInstr *NewMI = cloneInstr(In, UINT_MAX, 0);
updateInstruction(NewMI, i == 1, EpilogStage, 0, VRMap);
NewBB->push_back(NewMI);
+ LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = In;
}
}
@@ -343,14 +351,11 @@ void ModuloScheduleExpander::generateEpilog(
/// basic block with ToReg.
static void replaceRegUsesAfterLoop(unsigned FromReg, unsigned ToReg,
MachineBasicBlock *MBB,
- MachineRegisterInfo &MRI,
- LiveIntervals &LIS) {
+ MachineRegisterInfo &MRI) {
for (MachineOperand &O :
llvm::make_early_inc_range(MRI.use_operands(FromReg)))
if (O.getParent()->getParent() != MBB)
O.setReg(ToReg);
- if (!LIS.hasInterval(ToReg))
- LIS.createEmptyInterval(ToReg);
}
/// Return true if the register has a use that occurs outside the
@@ -541,8 +546,10 @@ void ModuloScheduleExpander::generateExistingPhis(
if (VRMap[LastStageNum - np - 1].count(LoopVal))
PhiOp2 = VRMap[LastStageNum - np - 1][LoopVal];
- if (IsLast && np == NumPhis - 1)
- replaceRegUsesAfterLoop(Def, NewReg, BB, MRI, LIS);
+ if (IsLast && np == NumPhis - 1) {
+ replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
+ NoIntervalRegs.push_back(NewReg);
+ }
continue;
}
}
@@ -560,6 +567,7 @@ void ModuloScheduleExpander::generateExistingPhis(
TII->get(TargetOpcode::PHI), NewReg);
NewPhi.addReg(PhiOp1).addMBB(BB1);
NewPhi.addReg(PhiOp2).addMBB(BB2);
+ LIS.InsertMachineInstrInMaps(*NewPhi);
if (np == 0)
InstrMap[NewPhi] = &*BBI;
@@ -581,8 +589,10 @@ void ModuloScheduleExpander::generateExistingPhis(
// Check if we need to rename any uses that occurs after the loop. The
// register to replace depends on whether the Phi is scheduled in the
// epilog.
- if (IsLast && np == NumPhis - 1)
- replaceRegUsesAfterLoop(Def, NewReg, BB, MRI, LIS);
+ if (IsLast && np == NumPhis - 1) {
+ replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
+ NoIntervalRegs.push_back(NewReg);
+ }
// In the kernel, a dependent Phi uses the value from this Phi.
if (InKernel)
@@ -600,9 +610,12 @@ void ModuloScheduleExpander::generateExistingPhis(
// Check if we need to rename a Phi that has been eliminated due to
// scheduling.
if (NumStages == 0 && IsLast) {
- auto It = VRMap[CurStageNum].find(LoopVal);
- if (It != VRMap[CurStageNum].end())
- replaceRegUsesAfterLoop(Def, It->second, BB, MRI, LIS);
+ auto &CurStageMap = VRMap[CurStageNum];
+ auto It = CurStageMap.find(LoopVal);
+ if (It != CurStageMap.end()) {
+ replaceRegUsesAfterLoop(Def, It->second, BB, MRI);
+ NoIntervalRegs.push_back(It->second);
+ }
}
}
}
@@ -702,6 +715,7 @@ void ModuloScheduleExpander::generatePhis(
TII->get(TargetOpcode::PHI), NewReg);
NewPhi.addReg(PhiOp1).addMBB(BB1);
NewPhi.addReg(PhiOp2).addMBB(BB2);
+ LIS.InsertMachineInstrInMaps(*NewPhi);
if (np == 0)
InstrMap[NewPhi] = &*BBI;
@@ -721,8 +735,10 @@ void ModuloScheduleExpander::generatePhis(
rewriteScheduledInstr(NewBB, InstrMap, CurStageNum, np, &*BBI, Def,
NewReg);
}
- if (IsLast && np == NumPhis - 1)
- replaceRegUsesAfterLoop(Def, NewReg, BB, MRI, LIS);
+ if (IsLast && np == NumPhis - 1) {
+ replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
+ NoIntervalRegs.push_back(NewReg);
+ }
}
}
}
@@ -831,9 +847,11 @@ void
ModuloScheduleExpander::splitLifetimes(MachineBasicBlock *KernelBB,
// We split the lifetime when we find the first use.
if (SplitReg == 0) {
SplitReg = MRI.createVirtualRegister(MRI.getRegClass(Def));
- BuildMI(*KernelBB, MI, MI->getDebugLoc(),
- TII->get(TargetOpcode::COPY), SplitReg)
- .addReg(Def);
+ MachineInstr *newCopy =
+ BuildMI(*KernelBB, MI, MI->getDebugLoc(),
+ TII->get(TargetOpcode::COPY), SplitReg)
+ .addReg(Def);
+ LIS.InsertMachineInstrInMaps(*newCopy);
}
BBJ.substituteRegister(Def, SplitReg, 0, *TRI);
}
@@ -901,6 +919,8 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock
&PreheaderBB,
removePhis(Epilog, LastEpi);
// Remove the blocks that are no longer referenced.
if (LastPro != LastEpi) {
+ for (auto &MI : *LastEpi)
+ LIS.RemoveMachineInstrFromMaps(MI);
LastEpi->clear();
LastEpi->eraseFromParent();
}
@@ -908,6 +928,8 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock
&PreheaderBB,
LoopInfo->disposed(&LIS);
NewKernel = nullptr;
}
+ for (auto &MI : *LastPro)
+ LIS.RemoveMachineInstrFromMaps(MI);
LastPro->clear();
LastPro->eraseFromParent();
} else {
@@ -928,6 +950,14 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock
&PreheaderBB,
}
}
+/// Some registers are generated during the kernel expansion. We calculate the
+/// live intervals of these registers after the expansion.
+void ModuloScheduleExpander::calculateIntervals() {
+ for (Register Reg : NoIntervalRegs)
+ LIS.createAndComputeVirtRegInterval(Reg);
+ NoIntervalRegs.clear();
+}
+
/// Return true if we can compute the amount the instruction changes
/// during each iteration. Set Delta to the amount of the change.
bool ModuloScheduleExpander::computeDelta(MachineInstr &MI, unsigned &Delta) {
@@ -1048,8 +1078,10 @@ void
ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
Register NewReg = MRI.createVirtualRegister(RC);
MO.setReg(NewReg);
VRMap[CurStageNum][reg] = NewReg;
- if (LastDef)
- replaceRegUsesAfterLoop(reg, NewReg, BB, MRI, LIS);
+ if (LastDef) {
+ replaceRegUsesAfterLoop(reg, NewReg, BB, MRI);
+ NoIntervalRegs.push_back(NewReg);
+ }
} else if (MO.isUse()) {
MachineInstr *Def = MRI.getVRegDef(reg);
// Compute the stage that contains the last definition for instruction.
@@ -1198,10 +1230,11 @@ void ModuloScheduleExpander::rewriteScheduledInstr(
UseOp.setReg(ReplaceReg);
else {
Register SplitReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg));
- BuildMI(*BB, UseMI, UseMI->getDebugLoc(), TII->get(TargetOpcode::COPY),
- SplitReg)
- .addReg(ReplaceReg);
+ MachineInstr *newCopy = BuildMI(*BB, UseMI, UseMI->getDebugLoc(),
+ TII->get(TargetOpcode::COPY), SplitReg)
+ .addReg(ReplaceReg);
UseOp.setReg(SplitReg);
+ LIS.InsertMachineInstrInMaps(*newCopy);
}
}
}
diff --git a/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals-issue128714.mir
b/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals-issue128714.mir
new file mode 100644
index 0000000000000..ef52ff11af9c8
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals-issue128714.mir
@@ -0,0 +1,157 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
UTC_ARGS: --version 5
+# RUN: llc --mtriple=hexagon %s -run-pass=pipeliner -o -| FileCheck %s
+
+--- |
+ define void @test_swp_ws_live_intervals(i32 %.pre) {
+ entry:
+ %cgep9 = bitcast ptr null to ptr
+ br label %for.body147
+
+ for.body147: ; preds = %for.body170,
%entry
+ %add11.i526 = or i32 %.pre, 1
+ br label %for.body158
+
+ for.body158: ; preds = %for.body158,
%for.body147
+ %lsr.iv = phi i32 [ %lsr.iv.next, %for.body158 ], [ -1, %for.body147 ]
+ %add11.i536602603 = phi i32 [ %add11.i526, %for.body147 ], [ 0,
%for.body158 ]
+ %and8.i534 = and i32 %add11.i536602603, 1
+ %cgep7 = getelementptr [64 x i32], ptr %cgep9, i32 0, i32 %and8.i534
+ store i32 0, ptr %cgep7, align 4
+ %lsr.iv.next = add nsw i32 %lsr.iv, 1
+ %cmp157.3 = icmp ult i32 %lsr.iv.next, 510
+ br i1 %cmp157.3, label %for.body158, label %for.body170
+
+ for.body170: ; preds = %for.body170,
%for.body158
+ %lsr.iv3 = phi ptr [ %cgep6, %for.body170 ], [ inttoptr (i32 4 to ptr),
%for.body158 ]
+ %lsr.iv1 = phi i32 [ %lsr.iv.next2, %for.body170 ], [ -1, %for.body158 ]
+ %add11.i556606607 = phi i32 [ 0, %for.body170 ], [ 1, %for.body158 ]
+ %cgep5 = getelementptr i8, ptr %lsr.iv3, i32 -4
+ store i32 0, ptr %cgep5, align 8
+ %sub.i547.1 = add i32 %add11.i556606607, 1
+ %and.i548.1 = and i32 %sub.i547.1, 1
+ %cgep8 = getelementptr [64 x i32], ptr %cgep9, i32 0, i32 %and.i548.1
+ %0 = load i32, ptr %cgep8, align 4
+ store i32 %0, ptr %lsr.iv3, align 4
+ %lsr.iv.next2 = add nsw i32 %lsr.iv1, 1
+ %cmp169.1 = icmp ult i32 %lsr.iv.next2, 254
+ %cgep6 = getelementptr i8, ptr %lsr.iv3, i32 2
+ br i1 %cmp169.1, label %for.body170, label %for.body147
+ }
+
+...
+---
+name: test_swp_ws_live_intervals
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_swp_ws_live_intervals
+ ; CHECK: bb.0.entry:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $r0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:intregs = COPY $r0
+ ; CHECK-NEXT: [[S2_setbit_i:%[0-9]+]]:intregs = S2_setbit_i [[COPY]], 0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[A2_andir:%[0-9]+]]:intregs = A2_andir [[S2_setbit_i]], 1
+ ; CHECK-NEXT: [[S2_asl_i_r:%[0-9]+]]:intregs = S2_asl_i_r [[A2_andir]], 2
+ ; CHECK-NEXT: [[A2_tfrsi:%[0-9]+]]:intregs = A2_tfrsi 1
+ ; CHECK-NEXT: [[A2_tfrsi1:%[0-9]+]]:intregs = A2_tfrsi 4
+ ; CHECK-NEXT: [[A2_tfrsi2:%[0-9]+]]:intregs = A2_tfrsi 0
+ ; CHECK-NEXT: J2_loop0i %bb.6, 510, implicit-def $lc0, implicit-def $sa0,
implicit-def $usr
+ ; CHECK-NEXT: J2_jump %bb.6, implicit-def $pc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.6(0x7c000000), %bb.7(0x04000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:intregs = PHI [[A2_tfrsi2]], %bb.5, %24,
%bb.6
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:intregs = PHI [[S2_asl_i_r]], %bb.5, %23,
%bb.6
+ ; CHECK-NEXT: S4_storeiri_io [[PHI1]], 0, 0 :: (store (s32) into %ir.cgep7)
+ ; CHECK-NEXT: [[A2_andir1:%[0-9]+]]:intregs = A2_andir [[PHI]], 1
+ ; CHECK-NEXT: [[A2_tfrsi3:%[0-9]+]]:intregs = A2_tfrsi 1
+ ; CHECK-NEXT: [[A2_tfrsi4:%[0-9]+]]:intregs = A2_tfrsi 4
+ ; CHECK-NEXT: [[S2_asl_i_r1:%[0-9]+]]:intregs = S2_asl_i_r [[A2_andir1]], 2
+ ; CHECK-NEXT: [[A2_tfrsi5:%[0-9]+]]:intregs = A2_tfrsi 0
+ ; CHECK-NEXT: ENDLOOP0 %bb.6, implicit-def $pc, implicit-def $lc0,
implicit $sa0, implicit $lc0
+ ; CHECK-NEXT: J2_jump %bb.7, implicit-def $pc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:intregs = PHI [[S2_asl_i_r1]], %bb.6
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:intregs = PHI [[A2_tfrsi3]], %bb.6
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:intregs = PHI [[A2_tfrsi4]], %bb.6
+ ; CHECK-NEXT: S4_storeiri_io [[PHI2]], 0, 0 :: (store unknown-size into
%ir.cgep7, align 4)
+ ; CHECK-NEXT: J2_jump %bb.3, implicit-def $pc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: J2_loop0i %bb.4, 255, implicit-def $lc0, implicit-def $sa0,
implicit-def $usr
+ ; CHECK-NEXT: J2_jump %bb.4, implicit-def $pc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.4(0x7c000000), %bb.1(0x04000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI5:%[0-9]+]]:intregs = PHI [[PHI4]], %bb.3, %9, %bb.4
+ ; CHECK-NEXT: [[PHI6:%[0-9]+]]:intregs = PHI [[PHI3]], %bb.3, %11, %bb.4
+ ; CHECK-NEXT: [[A2_tfrsi6:%[0-9]+]]:intregs = A2_tfrsi 0
+ ; CHECK-NEXT: S2_storeri_io [[PHI5]], -4, [[A2_tfrsi6]] :: (store (s32)
into %ir.cgep5, align 8)
+ ; CHECK-NEXT: [[A2_addi:%[0-9]+]]:intregs = A2_addi [[PHI6]], 1
+ ; CHECK-NEXT: [[S2_insert:%[0-9]+]]:intregs = S2_insert [[PHI2]],
[[A2_addi]], 1, 2
+ ; CHECK-NEXT: [[L2_loadri_io:%[0-9]+]]:intregs = L2_loadri_io
[[S2_insert]], 0 :: (load (s32) from %ir.cgep8)
+ ; CHECK-NEXT: S2_storeri_io [[PHI5]], 0, [[L2_loadri_io]] :: (store (s32)
into %ir.lsr.iv3)
+ ; CHECK-NEXT: [[A2_addi1:%[0-9]+]]:intregs = A2_addi [[PHI5]], 2
+ ; CHECK-NEXT: ENDLOOP0 %bb.4, implicit-def $pc, implicit-def $lc0,
implicit $sa0, implicit $lc0
+ ; CHECK-NEXT: J2_jump %bb.1, implicit-def dead $pc
+ bb.0.entry:
+ successors: %bb.1(0x80000000)
+ liveins: $r0
+
+ %0:intregs = COPY $r0
+ %1:intregs = S2_setbit_i %0, 0
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+
+ J2_loop0i %bb.2, 511, implicit-def $lc0, implicit-def $sa0, implicit-def
$usr
+
+ bb.2:
+ successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+
+ %2:intregs = PHI %1, %bb.1, %3, %bb.2
+ %4:intregs = A2_andir %2, 1
+ %5:intregs = S2_asl_i_r %4, 2
+ S4_storeiri_io %5, 0, 0 :: (store (s32) into %ir.cgep7)
+ %6:intregs = A2_tfrsi 1
+ %7:intregs = A2_tfrsi 4
+ %3:intregs = A2_tfrsi 0
+ ENDLOOP0 %bb.2, implicit-def $pc, implicit-def $lc0, implicit $sa0,
implicit $lc0
+ J2_jump %bb.3, implicit-def dead $pc
+
+ bb.3:
+ successors: %bb.4(0x80000000)
+
+ J2_loop0i %bb.4, 255, implicit-def $lc0, implicit-def $sa0, implicit-def
$usr
+ J2_jump %bb.4, implicit-def $pc
+
+ bb.4:
+ successors: %bb.4(0x7c000000), %bb.1(0x04000000)
+
+ %8:intregs = PHI %7, %bb.3, %9, %bb.4
+ %10:intregs = PHI %6, %bb.3, %11, %bb.4
+ %11:intregs = A2_tfrsi 0
+ S2_storeri_io %8, -4, %11 :: (store (s32) into %ir.cgep5, align 8)
+ %12:intregs = A2_addi %10, 1
+ %13:intregs = S2_insert %5, %12, 1, 2
+ %14:intregs = L2_loadri_io %13, 0 :: (load (s32) from %ir.cgep8)
+ S2_storeri_io %8, 0, %14 :: (store (s32) into %ir.lsr.iv3)
+ %9:intregs = A2_addi %8, 2
+ ENDLOOP0 %bb.4, implicit-def $pc, implicit-def $lc0, implicit $sa0,
implicit $lc0
+ J2_jump %bb.1, implicit-def dead $pc
+
+...
>From a141e58685fd8d07b13751dcce0d2fca27b93848 Mon Sep 17 00:00:00 2001
From: Hua Tian <akirat...@tencent.com>
Date: Thu, 3 Apr 2025 14:25:55 +0800
Subject: [PATCH 3/3] [llvm][CodeGen] avoid repeated interval calculation in
window scheduler (#132352)
Some new registers are reused when replacing some old ones in
certain use case of ModuloScheduleExpander. It is necessary to
avoid repeated interval calculations for these registers.
(cherry picked from commit 7e65944292278cc245e36cc6ca971654d584012d)
---
llvm/include/llvm/CodeGen/ModuloSchedule.h | 4 -
llvm/lib/CodeGen/ModuloSchedule.cpp | 32 +-----
.../AArch64/aarch64-swp-ws-live-intervals.mir | 103 ++++++++++++++++++
3 files changed, 108 insertions(+), 31 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h
b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index e2fb1daf9fb4e..64598ce449a44 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -188,9 +188,6 @@ class ModuloScheduleExpander {
/// Instructions to change when emitting the final schedule.
InstrChangesTy InstrChanges;
- /// Record the registers that need to compute live intervals.
- SmallVector<Register> NoIntervalRegs;
-
void generatePipelinedLoop();
void generateProlog(unsigned LastStage, MachineBasicBlock *KernelBB,
ValueMapTy *VRMap, MBBVectorTy &PrologBBs);
@@ -214,7 +211,6 @@ class ModuloScheduleExpander {
void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs,
MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs,
ValueMapTy *VRMap);
- void calculateIntervals();
bool computeDelta(MachineInstr &MI, unsigned &Delta);
void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI,
unsigned Num);
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp
b/llvm/lib/CodeGen/ModuloSchedule.cpp
index fcfb3c1f7073d..7792a0eaa285b 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -181,10 +181,6 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
// Add branches between prolog and epilog blocks.
addBranches(*Preheader, PrologBBs, KernelBB, EpilogBBs, VRMap);
- // The intervals of newly created virtual registers are calculated after the
- // kernel expansion.
- calculateIntervals();
-
delete[] VRMap;
delete[] VRMapPhi;
}
@@ -546,10 +542,8 @@ void ModuloScheduleExpander::generateExistingPhis(
if (VRMap[LastStageNum - np - 1].count(LoopVal))
PhiOp2 = VRMap[LastStageNum - np - 1][LoopVal];
- if (IsLast && np == NumPhis - 1) {
+ if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
- }
continue;
}
}
@@ -589,10 +583,8 @@ void ModuloScheduleExpander::generateExistingPhis(
// Check if we need to rename any uses that occurs after the loop. The
// register to replace depends on whether the Phi is scheduled in the
// epilog.
- if (IsLast && np == NumPhis - 1) {
+ if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
- }
// In the kernel, a dependent Phi uses the value from this Phi.
if (InKernel)
@@ -612,10 +604,8 @@ void ModuloScheduleExpander::generateExistingPhis(
if (NumStages == 0 && IsLast) {
auto &CurStageMap = VRMap[CurStageNum];
auto It = CurStageMap.find(LoopVal);
- if (It != CurStageMap.end()) {
+ if (It != CurStageMap.end())
replaceRegUsesAfterLoop(Def, It->second, BB, MRI);
- NoIntervalRegs.push_back(It->second);
- }
}
}
}
@@ -735,10 +725,8 @@ void ModuloScheduleExpander::generatePhis(
rewriteScheduledInstr(NewBB, InstrMap, CurStageNum, np, &*BBI, Def,
NewReg);
}
- if (IsLast && np == NumPhis - 1) {
+ if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
- }
}
}
}
@@ -950,14 +938,6 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock
&PreheaderBB,
}
}
-/// Some registers are generated during the kernel expansion. We calculate the
-/// live intervals of these registers after the expansion.
-void ModuloScheduleExpander::calculateIntervals() {
- for (Register Reg : NoIntervalRegs)
- LIS.createAndComputeVirtRegInterval(Reg);
- NoIntervalRegs.clear();
-}
-
/// Return true if we can compute the amount the instruction changes
/// during each iteration. Set Delta to the amount of the change.
bool ModuloScheduleExpander::computeDelta(MachineInstr &MI, unsigned &Delta) {
@@ -1078,10 +1058,8 @@ void
ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
Register NewReg = MRI.createVirtualRegister(RC);
MO.setReg(NewReg);
VRMap[CurStageNum][reg] = NewReg;
- if (LastDef) {
+ if (LastDef)
replaceRegUsesAfterLoop(reg, NewReg, BB, MRI);
- NoIntervalRegs.push_back(NewReg);
- }
} else if (MO.isUse()) {
MachineInstr *Def = MRI.getVRegDef(reg);
// Compute the stage that contains the last definition for instruction.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
b/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
new file mode 100644
index 0000000000000..48f02452e3597
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
@@ -0,0 +1,103 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
UTC_ARGS: --version 5
+# RUN: llc --mtriple=aarch64 %s -run-pass=pipeliner -o - | FileCheck %s
+
+...
+---
+name: foo
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: foo
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+ ; CHECK-NEXT: [[FMOVD0_:%[0-9]+]]:fpr64 = FMOVD0
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64sp = SUBREG_TO_REG 0,
[[MOVi32imm]], %subreg.sub_32
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[FADDDrr:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FMOVD0_]],
[[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[SUBSXri:%[0-9]+]]:gpr64 = nsw SUBSXri [[SUBREG_TO_REG]],
1, 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY [[SUBSXri]]
+ ; CHECK-NEXT: [[FMOVDi:%[0-9]+]]:fpr64 = FMOVDi 112
+ ; CHECK-NEXT: Bcc 0, %bb.7, implicit $nzcv
+ ; CHECK-NEXT: B %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000), %bb.6(0x00000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[FADDDrr1:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FADDDrr]],
[[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDDrr2:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FMOVD0_]],
[[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[SUBSXri1:%[0-9]+]]:gpr64 = nsw SUBSXri [[COPY1]], 1, 0,
implicit-def $nzcv
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64all = COPY [[SUBSXri1]]
+ ; CHECK-NEXT: [[FMOVDi1:%[0-9]+]]:fpr64 = FMOVDi 112
+ ; CHECK-NEXT: Bcc 0, %bb.6, implicit $nzcv
+ ; CHECK-NEXT: B %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x04000000), %bb.5(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64sp = PHI [[COPY2]], %bb.4, %24, %bb.5
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:fpr64 = PHI [[FMOVDi1]], %bb.4, %25, %bb.5
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:fpr64 = PHI [[FMOVDi]], %bb.4, [[PHI1]],
%bb.5
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:fpr64 = PHI [[FADDDrr2]], %bb.4, %22, %bb.5
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:fpr64 = PHI [[FADDDrr1]], %bb.4, %23, %bb.5
+ ; CHECK-NEXT: [[SUBSXri2:%[0-9]+]]:gpr64 = nsw SUBSXri [[PHI]], 1, 0,
implicit-def $nzcv
+ ; CHECK-NEXT: [[FADDDrr3:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI2]],
[[FMOVD0_]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDDrr4:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI3]],
[[PHI2]], implicit $fpcr
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64all = COPY [[SUBSXri2]]
+ ; CHECK-NEXT: STRDui [[PHI4]], [[COPY]], 0
+ ; CHECK-NEXT: [[FMOVDi2:%[0-9]+]]:fpr64 = FMOVDi 112
+ ; CHECK-NEXT: Bcc 1, %bb.5, implicit $nzcv
+ ; CHECK-NEXT: B %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI5:%[0-9]+]]:fpr64 = PHI [[FMOVDi]], %bb.4, [[PHI1]],
%bb.5
+ ; CHECK-NEXT: [[PHI6:%[0-9]+]]:fpr64 = PHI [[FADDDrr2]], %bb.4,
[[FADDDrr3]], %bb.5
+ ; CHECK-NEXT: [[PHI7:%[0-9]+]]:fpr64 = PHI [[FADDDrr1]], %bb.4,
[[FADDDrr4]], %bb.5
+ ; CHECK-NEXT: STRDui [[PHI7]], [[COPY]], 0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI8:%[0-9]+]]:fpr64 = PHI [[FMOVD0_]], %bb.3, [[PHI5]],
%bb.6
+ ; CHECK-NEXT: [[PHI9:%[0-9]+]]:fpr64 = PHI [[FADDDrr]], %bb.3, [[PHI6]],
%bb.6
+ ; CHECK-NEXT: [[FADDDrr5:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI9]],
[[PHI8]], implicit $fpcr
+ ; CHECK-NEXT: STRDui [[FADDDrr5]], [[COPY]], 0
+ ; CHECK-NEXT: B %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $x0
+
+ %0:gpr64common = COPY $x0
+ %1:fpr64 = FMOVD0
+ %2:gpr32 = MOVi32imm 1
+ %3:gpr64all = SUBREG_TO_REG 0, killed %2, %subreg.sub_32
+
+ bb.1:
+ successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+ %4:gpr64sp = PHI %3, %bb.0, %5, %bb.1
+ %6:fpr64 = PHI %1, %bb.0, %7, %bb.1
+ %8:fpr64 = PHI %1, %bb.0, %6, %bb.1
+ %9:fpr64 = nofpexcept FADDDrr %8, %1, implicit $fpcr
+ %10:fpr64 = nofpexcept FADDDrr killed %9, %6, implicit $fpcr
+ STRDui killed %10, %0, 0
+ %11:gpr64 = nsw SUBSXri %4, 1, 0, implicit-def $nzcv
+ %5:gpr64all = COPY %11
+ %7:fpr64 = FMOVDi 112
+ Bcc 1, %bb.1, implicit $nzcv
+ B %bb.2
+
+ bb.2:
+ RET_ReallyLR
+
+...
_______________________________________________
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
