https://github.com/Pierre-vh updated https://github.com/llvm/llvm-project/pull/232344
>From 6c1a8ed38667403af7b736fd69894c80b76b8ef2 Mon Sep 17 00:00:00 2001 From: pvanhout <[email protected]> Date: Thu, 15 Oct 2026 16:05:30 +0200 Subject: [PATCH] [AMDGPU][NewInsertWaitcnt] Event Tracker (2/2) See #226970 for context and history regarding this patch. This is intended to begin review as a 1-1 equivalent of #226970 with only one change: a "Drain" test that Claude added and I found useful, so I left it in. Builds upon the previous patch to include the functionality needed to import events from predecessor basic blocks into a basic block, so a MBB can be aware of previous state. This adds the "age" system, which is necessary to preserve chronology across control-flow edges, especially in asymmetrical diamonds, alongside a way to import predecessor state via a `enterBlock` function. Assisted-by: AI Coding Assistants for basic boilerplate/research/etc. - no logic was written by LLMs except some unit tests. --- .../lib/Target/AMDGPU/AMDGPUEventTracking.cpp | 193 ++++++- llvm/lib/Target/AMDGPU/AMDGPUEventTracking.h | 128 ++++- .../Target/AMDGPU/AMDGPUEventTrackingTest.cpp | 474 ++++++++++++++++++ 3 files changed, 767 insertions(+), 28 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.cpp b/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.cpp index b806e70b95f2e4..d285482bb6e0b4 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.cpp @@ -32,13 +32,17 @@ namespace eventtracking { void EventTrackerRecord::print(raw_ostream &OS, bool PrintMI, unsigned Indent) const { - OS.indent(Indent) << "Record Kind=" << Kind << ": "; + OS.indent(Indent) << "Record Kind=" << Kind << " Age=" << Age << ": "; if (PrintMI && MI) OS << *MI; else OS << "MI@" << (void *)MI << '\n'; } +hash_code EventTrackerRecord::getIdentity() const { + return hash_combine((void *)MI, Kind.rawValue()); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void EventTrackerRecord::dump() const { dbgs() << '\n'; @@ -55,6 +59,36 @@ EventTracker::EventTracker(const MachineBasicBlock &MBB, Counters[Info.CounterT].CI = &Info; } +void EventTracker::enterBlock(GetEventTrackerFn EventTrackerGetter) { + LLVM_DEBUG(dbgs() << "\n[EventTracker] Entering "; + MBB->printAsOperand(dbgs()); dbgs() << '\n'); + + // FIXME: This is a bit hacky, but we need to save the old state in case the + // MBB is also its own predecessor. Revisit when the design and clients of + // this class are set in stone. + + SmallVector<EventTracker *> Preds; + bool IsSelfPred = false; + if (!MBB->pred_empty()) { + for (MachineBasicBlock *Pred : MBB->predecessors()) { + if (Pred == MBB) + IsSelfPred = true; + else + Preds.push_back(&EventTrackerGetter(*Pred)); + } + } + + if (IsSelfPred) { + EventTracker SelfCopy = *this; + Preds.push_back(&SelfCopy); + clear(); + recordIncomings(Preds); + } else { + clear(); + recordIncomings(Preds); + } +} + void EventTracker::record(MachineInstr &MI, SingleHWEvent Event) { LLVM_DEBUG(dbgs() << "[EventTracker] Recording " << Event << ": " << MI); @@ -66,6 +100,27 @@ void EventTracker::record(MachineInstr &MI, SingleHWEvent Event) { FoundMatch = true; ++CD.Count; + + // NB: There is an intentional tradeoff here. We could avoid this loop by + // instead storing a timestamp in each record, and having a monotonically + // increasing clock to infer the age (clock - timestamp = age). + // However, it would: + // - Complicate fetching the age (`EventTrackerRecord` could no longer + // answer it on its own, and we would need a separate query/wrapper). + // - Make merging incoming records more annoying (we would need to rebase + // the clock). + // - Potentially require (much) more space in EventTrackerRecord to store + // bigger numbers. + // + // All in all, this small loop is fine for now, but we can revisit it if + // profiling data justifies the change. + for (auto &Live : CD.Timeline) + Live.setAge(Live.getAge() + 1); + + // NOTE: Unlike recordIncomings, we do not merge with a previous record that + // has the same identity (MI+Kind). Having two such records is fine: one is + // carried over from a backedge and the other is from a more recent + // iteration. CD.Timeline.push_back(Rec); LLVM_DEBUG(if (EventTrackerPrintAll) { @@ -93,16 +148,13 @@ void EventTracker::drain(InstCounterType T, unsigned N) { } else { CD.Count = std::min(CD.Count, N); - // Only keep the N youngest records. - if (CD.Timeline.size() > N) { - auto *RmEnd = CD.Timeline.end() - N; - LLVM_DEBUG(for (const EventTrackerRecord &E - : make_range(CD.Timeline.begin(), RmEnd)) { - dbgs() << " | Removing "; - E.print(dbgs()); - }); - CD.Timeline.erase(CD.Timeline.begin(), RmEnd); - } + auto *RmIt = remove_if(CD.Timeline, [&](EventTrackerRecord &E) { + if (E.getAge() < N) + return false; + LLVM_DEBUG(dbgs() << " | Removing "; E.print(dbgs())); + return true; + }); + CD.Timeline.erase(RmIt, CD.Timeline.end()); LLVM_DEBUG(dbgs() << " | => Updated Count:" << CD.Count << '\n'); @@ -148,10 +200,37 @@ void EventTracker::verify() const { print(dbgs(), C); }; - // The timeline cannot have more records than the value of the counter. - if (C.Timeline.size() > C.Count) { + // Check some basic invariants: + // - The age of a record cannot exceed the value of the counter. + // - We cannot have two records with the same identity at the same age, + // e.g. two VMEM_READ_ACCESS records for the same instruction at the + // same age. `recordIncomings` merges based on identity, and `record` + // increments the age of all pre-existing records before inserting a + // new one. + DenseSet<std::pair<hash_code, unsigned>> RecIdentityCheck; + for (const EventTrackerRecord &E : C.Timeline) { + if (E.getAge() > C.Count) { + OnError(); + dbgs() << "Concerning Record:"; + E.print(dbgs()); + llvm_unreachable("record age is out of range"); + } + + auto [It, Inserted] = + RecIdentityCheck.insert({E.getIdentity(), E.getAge()}); + if (!Inserted) { + OnError(); + dbgs() << "Concerning Record:"; + E.print(dbgs()); + llvm_unreachable("The timeline cannot have two records with the same " + "MachineInstr, Kind and Age at the same time!"); + } + } + + // Check that the timeline is sorted. + if (!is_sorted(C.Timeline, compareRecords)) { OnError(); - llvm_unreachable("timeline has more records than the counter value"); + llvm_unreachable("live records are not sorted!"); } } } @@ -185,6 +264,92 @@ LLVM_DUMP_METHOD void EventTracker::dump() const { } #endif +bool EventTracker::compareRecords(const EventTrackerRecord &A, + const EventTrackerRecord &B) { + // Different age. + if (A.getAge() != B.getAge()) + return A.getAge() > B.getAge(); + + // Same age but different kind. + unsigned AKindV = A.getKind().rawValue(); + unsigned BKindV = B.getKind().rawValue(); + if (AKindV != BKindV) + return AKindV > BKindV; + + const MachineInstr *AMI = A.getMI(); + const MachineInstr *BMI = B.getMI(); + if (AMI == BMI) + return false; + + // Same age/kind but MIs are in different MBBs: the one in the MBB with the + // lower number comes first. + const MachineBasicBlock *Bbb = BMI->getParent(); + const MachineBasicBlock *Abb = AMI->getParent(); + if (Abb != Bbb) + return Abb->getNumber() < Bbb->getNumber(); + + llvm_unreachable("We cannot have two distinct instructions from the same " + "MBB, at the same age and with the same event kind!"); +} + +void EventTracker::clear() { + for (CounterData &C : Counters) { + C.Timeline.clear(); + C.Count = 0; + } +} + +void EventTracker::recordIncomings(ArrayRef<EventTracker *> Preds) { + LLVM_DEBUG(if (!Preds.empty()) { + dbgs() << "[EventTracker] Recording incoming events (merge) from " + "predecessors:\n"; + for (EventTracker *Pred : Preds) { + Pred->print(dbgs(), /*IgnoreEmpty=*/true, /*Indent=*/2); + } + }); + + // Iterate over all counters that are available to us. + for (CounterData &CData : Counters) { + assert(CData.Timeline.empty()); + + DenseMap<hash_code, EventTrackerRecord> Acc; + + for (EventTracker *Pred : Preds) { + auto &PredCData = Pred->Counters[CData.CI->CounterT]; + + // The merged count is the max across predecessors. + CData.Count = std::max(CData.Count, PredCData.Count); + + for (EventTrackerRecord &PredEntry : PredCData.Timeline) { + // At a join, collapse records from all predecessors with the same + // identity (MI+Kind) into a single record whose age is the minimum + // across predecessors. + auto [It, Inserted] = + Acc.try_emplace(PredEntry.getIdentity(), PredEntry); + if (!Inserted) { + auto &AccVal = It->second; + AccVal.setAge(std::min(AccVal.getAge(), PredEntry.getAge())); + } + } + } + + auto AccVals = Acc.values(); + CData.Timeline.append(AccVals.begin(), AccVals.end()); + + // Sort records by descending age for consistent iteration. + stable_sort(CData.Timeline, compareRecords); + } + +#ifdef EXPENSIVE_CHECKS + verify(); +#endif + + LLVM_DEBUG(if (!Preds.empty()) { + dbgs() << "[EventTracker] Timeline after recording incomings:\n"; + print(dbgs(), /*IgnoreEmpty=*/true, /*Indent=*/2); + }); +} + void EventTracker::print(raw_ostream &OS, const CounterData &CD, unsigned Indent) { OS.indent(Indent) << getInstCounterName(CD.CI->CounterT) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.h b/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.h index 1aba3af3d48616..b9b6ddce89be50 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUEventTracking.h @@ -13,11 +13,16 @@ /// outstanding and how far back they are, but does not directly compute waits. /// /// - \ref CounterInfo: constant per-counter description (events, limit). -/// - \ref EventTrackerRecord: a (MachineInstr, SingleHWEvent) pair. +/// - \ref EventTrackerRecord: a (MachineInstr, SingleHWEvent) pair with a +/// age. A wait N, where N is <= to the age of the record, guarantees that +/// event has completed. /// - \ref EventTracker: per-MBB state; for each counter, a count and a -/// sequence of records ordered from oldest to youngest. Driven by the -/// client while visiting the MBB: \ref EventTracker::record appends new -/// events, and \ref EventTracker::drain removes oldest events. +/// timeline of records sorted by descending age. Driven by the client +/// during an RPO dataflow analysis: \ref EventTracker::enterBlock merges +/// predecessor state, \ref EventTracker::record appends new events and +/// increments the age of all pre-existing events, and +/// \ref EventTracker::drain removes events that are older than a given +/// value. /// /// This is a low-level API. It can be used directly for trivial waitcnt /// calculation (e.g. wait on a specific MI), but the typical usage is for @@ -42,6 +47,8 @@ class MachineBasicBlock; namespace AMDGPU { namespace eventtracking { +class EventTracker; + /// FIXME: Move this to a common file and make it a generic utility. /// /// Represents target-specific information about an available @@ -63,11 +70,41 @@ struct CounterInfo { /// Represents a record within the \ref EventTracker's timeline /// for one \ref InstCounterType. A record is a pair of \ref MachineInstr -/// and \ref SingleHWEvent. +/// and \ref SingleHWEvent with an age in the timeline. +/// +/// The age is a conservative minimum estimate of *when* the event it +/// represents occurred. It is used to emit a wait instruction that ensures the +/// event has completed (e.g. Age = 2 means "wait 2" on the counter ensures +/// completion of the event across all control-flow paths). +/// +/// Two examples illustrate how the age works: +/// +/// If we are iterating a single block where there are no records inherited from +/// incoming basic blocks, then the age of a record is trivial: if there are +/// N records, the oldest is at age N-1, while the youngest is at age 0. +/// The age of each existing record increases whenever a new one is added. +/// +/// However, if one record is carried into multiple successor blocks and merged +/// later (e.g. in a diamond), then the divergence (the same record could be at +/// a different age depending on the CFG path taken) is reconciled by merging +/// the records with the same \ref MachineInstr and \ref SingleHWEvent into a +/// single record whose age is the minimum across the predecessors. +/// +/// The second example shows that the age is decoupled from a record's +/// position in the timeline. For example, a timeline could have 4 records, 3 +/// of them at age 0 and the remaining one at age 1. +/// +/// The primary motivation for this system is that we want the capability to +/// determine what kind of waitcnt must be emitted for a given event (record) to +/// complete. For example, let R be a record in the timeline of a counter T, +/// then a "waitcnt_T N" (a "drain") where N is <= the age of R must be +/// sufficient to guarantee that R has completed (and its results can be used). class EventTrackerRecord { public: - EventTrackerRecord(MachineInstr *MI, SingleHWEvent Kind) - : MI(MI), Kind(Kind) {} + EventTrackerRecord(MachineInstr *MI, SingleHWEvent Kind, uint32_t Age = 0) + : MI(MI), Kind(Kind) { + setAge(Age); + } /// \returns the MachineInstr that originated this record. MachineInstr *getMI() const { return MI; } @@ -75,15 +112,35 @@ class EventTrackerRecord { /// \returns the kind of record this is, as a \ref SingleHWEvent. SingleHWEvent getKind() const { return Kind; } + /// \returns the age of this record. + uint32_t getAge() const { return Age; } + void print(raw_ostream &OS, bool PrintMI = true, unsigned Indent = 0) const; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void dump() const; #endif + /// \returns the "identity" of this record, which is a hash of its kind and + /// MI. + hash_code getIdentity() const; + private: + friend class EventTracker; + + /// Sets the age of this record to \p NewAge. + void setAge(uint32_t NewAge) { + Age = NewAge; + assert(Age == NewAge && "Age overflow!"); + } + MachineInstr *MI; SingleHWEvent Kind; + // Age should never exceed the uint8_t limit in normal circumstances, as + // most counter types use at most 6 bits to encode waitcnts. 16 bits is a + // very generous limit; we can probably shrink it once we refine how we + // handle overflowing inst counters. + uint16_t Age; }; /// This assert serves as a reminder to be mindful of the size of the object. @@ -101,16 +158,26 @@ static_assert(sizeof(EventTrackerRecord) == 16, /// (e.g. finding an aliasing store). These responsibilities are left to /// helpers built on top of this class (see the file header). /// -/// The expected usage pattern is to visit the MBB in instruction order, call +/// It is intended to be used during a reverse postorder dataflow analysis of a +/// function. The expected usage pattern is to call +/// \ref EventTracker::enterBlock, then visit the MBB in instruction order, call /// \ref EventTracker::record whenever an interesting event occurs, and call /// \ref EventTracker::drain whenever a wait on a counter is performed. /// In between those operations, the user of the class can use all other methods /// to query the current state of a counter, iterate its timeline, etc. /// /// Allocation/storage/mapping of EventTrackers to MBBs is left to the user of -/// the class. +/// the class via \ref EventTracker::GetEventTrackerFn. +/// +/// Whenever the dataflow analysis re-visits a block, the user of the class is +/// expected to re-use the same EventTracker (which is required in case the MBB +/// is its own predecessor) and call \ref EventTracker::enterBlock again, which +/// clears the state and re-imports all incoming state. class EventTracker { public: + using GetEventTrackerFn = + function_ref<EventTracker &(const MachineBasicBlock &)>; + /// \p MBB is the MachineBasicBlock for this tracker. /// \p CounterInfos describes the available \ref InstCounterType. It must /// contain exactly one entry for each counter type in @@ -119,6 +186,16 @@ class EventTracker { EventTracker(const MachineBasicBlock &MBB, ArrayRef<CounterInfo> CounterInfos); + /// Prepare this EventTracker for iteration through its basic block. + /// + /// Clears the current state and merges the state of the EventTrackers of + /// predecessor MBBs. Records are merged using their identity (see + /// \ref EventTrackerRecord::getIdentity) and only the lowest age is kept. + /// Counts are merged by taking the maximum. + /// + /// \param EventTrackerGetter maps a MachineBasicBlock to its EventTracker. + void enterBlock(GetEventTrackerFn EventTrackerGetter); + /// Record an event of type \p Event at a MachineInstr \p MI, which will /// affect all counters that have \p Event in their event set. /// \pre \p Event belongs to at least one \ref InstCounterType. @@ -126,19 +203,21 @@ class EventTracker { /// Drains the timeline of \p T such that: /// - The \ref count of \p T is no higher than \p N. - /// - Only the \p N youngest records remain in the timeline. + /// - There are no records in the timeline with an age >= \p N. void drain(InstCounterType T, unsigned N = 0); - /// \returns a conservative estimate of the current value of the counter \p T. + /// \returns a conservative estimate of the current value of the counter \p T + /// (a maximum value across all possible CFG paths). unsigned count(InstCounterType T) const; /// \returns the set of pending HWEvents for \p T. HWEvents getPendingEvents(InstCounterType T) const; /// \returns the current timeline for \p T, which contains all known in-flight - /// (not drained) records, ordered from oldest to youngest. + /// (not drained) records. The timeline is always sorted by descending + /// age. /// The returned ArrayRef is invalidated whenever the timeline changes (e.g. - /// \ref record, \ref drain). + /// \ref record, \ref drain, \ref enterBlock). ArrayRef<EventTrackerRecord> getTimeline(InstCounterType T) const; /// Prints a dump of the internal tracking state of this class for \p T to the @@ -168,13 +247,34 @@ class EventTracker { SmallVector<EventTrackerRecord, 16> Timeline; - /// The current value of the counter. + /// The current value of the counter. This is a max (upper bound) across + /// all possible execution paths at runtime. It cannot be inferred from the + /// Timeline alone and is thus a separate tracking domain. /// /// FIXME: We shouldn't need it for correctness so perhaps it should be /// removed entirely, or be marked as debug. uint32_t Count = 0; + + // TODO: We could imagine storing the per-predecessor age for incoming + // events. We could achieve that by storing that as a map of ((ID, Pred), + // Age). This would allow identifying events that are "deep" in one + // branch but "shallow" in another, e.g. an event needing a waitcnt 1 for + // one pred, but a waitcnt 8 for another. Not sure if we can exploit that + // though? }; + /// Orders records by descending age. Ties are broken by kind, then by + /// parent MBB number, so that the order is deterministic. + static bool compareRecords(const EventTrackerRecord &A, + const EventTrackerRecord &B); + + /// Clears the tracked data, used when entering a block. + void clear(); + + /// Import all records from the predecessor trackers in \p Preds and + /// reconcile divergence at joins. + void recordIncomings(ArrayRef<EventTracker *> Preds); + static void print(raw_ostream &OS, const CounterData &CD, unsigned Indent = 0); diff --git a/llvm/unittests/Target/AMDGPU/AMDGPUEventTrackingTest.cpp b/llvm/unittests/Target/AMDGPU/AMDGPUEventTrackingTest.cpp index 92ce5fa57bfefe..8bd4e8d450f844 100644 --- a/llvm/unittests/Target/AMDGPU/AMDGPUEventTrackingTest.cpp +++ b/llvm/unittests/Target/AMDGPU/AMDGPUEventTrackingTest.cpp @@ -52,6 +52,7 @@ class AMDGPUGFX12EventTrackingTest : public AMDGPUCodeGenTestBase { const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>(); EventTracker &ET = TrackerGetter(MBB); + ET.enterBlock(TrackerGetter); for (MachineInstr &MI : MBB) { HWEvents Events = getEventsFor(MI, ST, /*IsExpertMode=*/false, /*TgSplit=*/false); @@ -112,6 +113,7 @@ TEST_F(AMDGPUGFX12EventTrackingTest, BasicTimeline) { name: BasicTimeline body: | bb.0: + liveins: $vgpr0_vgpr1, $vgpr1_vgpr2, $vgpr3_vgpr4, $vgpr4_vgpr5 GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec $vgpr4 = GLOBAL_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec @@ -130,12 +132,14 @@ body: | EXPECT_EQ(LoadCnt.getCount(), 1u); EXPECT_FALSE(LoadCnt.empty()); EXPECT_EQ(LoadCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_LOAD_DWORD); + EXPECT_EQ(LoadCnt.cur().getAge(), 0u); EXPECT_FALSE(LoadCnt.next()); auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); EXPECT_EQ(DsCnt.getCount(), 1u); EXPECT_FALSE(DsCnt.empty()); EXPECT_EQ(DsCnt.cur().getMI()->getOpcode(), AMDGPU::DS_READ_B32_gfx9); + EXPECT_EQ(DsCnt.cur().getAge(), 0u); EXPECT_FALSE(DsCnt.next()); auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); @@ -145,8 +149,10 @@ body: | EXPECT_EQ(StoreCnt.getCount(), 2u); EXPECT_FALSE(StoreCnt.empty()); EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getAge(), 1u); EXPECT_TRUE(StoreCnt.next()); EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); EXPECT_FALSE(StoreCnt.next()); } @@ -156,6 +162,7 @@ TEST_F(AMDGPUGFX12EventTrackingTest, Drain) { name: Drain body: | bb.0: + liveins: $vgpr0_vgpr1, $vgpr1_vgpr2, $vgpr3_vgpr4, $vgpr4_vgpr5 GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec GLOBAL_STORE_DWORDX2 $vgpr1_vgpr2, $vgpr3_vgpr4, 0, 0, implicit $exec @@ -184,8 +191,12 @@ body: | EXPECT_FALSE(StoreCnt.empty()); EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), + 1u); // drain doesn't change record ages EXPECT_TRUE(StoreCnt.next()); EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getAge(), + 0u); // drain doesn't change record ages EXPECT_FALSE(StoreCnt.next()); } @@ -204,4 +215,467 @@ body: | EXPECT_FALSE(ET.getPendingEvents(AMDGPU::STORE_CNT)); } +/// Check cases where a block loops back on itself. +TEST_F(AMDGPUGFX12EventTrackingTest, SelfPredecessor) { + StringRef MIR = R"( +name: BasicTimeline +body: | + bb.0: + liveins: $vgpr1_vgpr2, $vgpr3_vgpr4, $vgpr4_vgpr5, $scc + + $vgpr4 = GLOBAL_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec + $vgpr0 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec + GLOBAL_STORE_DWORDX2 $vgpr1_vgpr2, $vgpr3_vgpr4, 0, 0, implicit $exec + S_CBRANCH_SCC1 %bb.0, implicit $scc + S_BRANCH %bb.1 + + bb.1: + S_ENDPGM 0 +... +)"; + ASSERT_TRUE(parseMIR(MIR)); + MachineFunction &MF = getMF("BasicTimeline"); + MachineBasicBlock &BB0 = *MF.getBlockNumbered(0); + + // Iterate twice + visitAll(BB0); + auto &ET = visitAll(BB0); + + auto LoadCnt = TrackerRecordsChecker(ET, AMDGPU::LOAD_CNT); + EXPECT_EQ(LoadCnt.getCount(), 2u); + EXPECT_FALSE(LoadCnt.empty()); + EXPECT_EQ(LoadCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_LOAD_DWORD); + EXPECT_EQ(LoadCnt.cur().getAge(), 1u); + EXPECT_TRUE(LoadCnt.next()); + EXPECT_EQ(LoadCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_LOAD_DWORD); + EXPECT_EQ(LoadCnt.cur().getAge(), 0u); + EXPECT_FALSE(LoadCnt.next()); + + auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); + EXPECT_EQ(DsCnt.getCount(), 2u); + EXPECT_FALSE(DsCnt.empty()); + EXPECT_EQ(DsCnt.cur().getMI()->getOpcode(), AMDGPU::DS_READ_B32_gfx9); + EXPECT_EQ(DsCnt.cur().getAge(), 1u); + EXPECT_TRUE(DsCnt.next()); + EXPECT_EQ(DsCnt.cur().getMI()->getOpcode(), AMDGPU::DS_READ_B32_gfx9); + EXPECT_EQ(DsCnt.cur().getAge(), 0u); + EXPECT_FALSE(DsCnt.next()); + + auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); + EXPECT_TRUE(ExpCnt.unused()); + + auto StoreCnt = TrackerRecordsChecker(ET, AMDGPU::STORE_CNT); + EXPECT_EQ(StoreCnt.getCount(), 2u); + EXPECT_FALSE(StoreCnt.empty()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), 1u); + EXPECT_TRUE(StoreCnt.next()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_FALSE(StoreCnt.next()); +} + +/// Check all events carry into the next incoming block. +TEST_F(AMDGPUGFX12EventTrackingTest, SingleIncomingBlock) { + StringRef MIR = R"( +name: SingleIncomingBlock +body: | + bb.0: + successors: %bb.1 + liveins: $vgpr0_vgpr1, $vgpr1_vgpr2, $vgpr3_vgpr4, $vgpr4_vgpr5 + + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + $vgpr4 = GLOBAL_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec + $vgpr0 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec + GLOBAL_STORE_DWORDX2 $vgpr1_vgpr2, $vgpr3_vgpr4, 0, 0, implicit $exec + + bb.1: + + S_ENDPGM 0 +... +)"; + ASSERT_TRUE(parseMIR(MIR)); + MachineFunction &MF = getMF("SingleIncomingBlock"); + MachineBasicBlock &BB0 = *MF.getBlockNumbered(0); + MachineBasicBlock &BB1 = *MF.getBlockNumbered(1); + + visitAll(BB0); + auto &ET = visitAll(BB1); + + auto LoadCnt = TrackerRecordsChecker(ET, AMDGPU::LOAD_CNT); + EXPECT_EQ(LoadCnt.getCount(), 1u); + EXPECT_FALSE(LoadCnt.empty()); + EXPECT_EQ(LoadCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_LOAD_DWORD); + EXPECT_EQ(LoadCnt.cur().getAge(), 0u); + EXPECT_FALSE(LoadCnt.next()); + + auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); + EXPECT_EQ(DsCnt.getCount(), 1u); + EXPECT_FALSE(DsCnt.empty()); + EXPECT_EQ(DsCnt.cur().getMI()->getOpcode(), AMDGPU::DS_READ_B32_gfx9); + EXPECT_EQ(DsCnt.cur().getAge(), 0u); + EXPECT_FALSE(DsCnt.next()); + + auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); + EXPECT_TRUE(ExpCnt.unused()); + + auto StoreCnt = TrackerRecordsChecker(ET, AMDGPU::STORE_CNT); + EXPECT_EQ(StoreCnt.getCount(), 2u); + EXPECT_FALSE(StoreCnt.empty()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getAge(), 1u); + EXPECT_TRUE(StoreCnt.next()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_FALSE(StoreCnt.next()); +} + +/// Check a simple merge where all events differ in each incoming block. +TEST_F(AMDGPUGFX12EventTrackingTest, SimpleDisjointMerge) { + StringRef MIR = R"( +name: SimpleDisjointMerge +body: | + bb.0: + successors: %bb.2 + liveins: $vgpr1, $vgpr4_vgpr5 + + $vgpr0 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec + $vgpr4 = GLOBAL_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec + S_BRANCH %bb.2 + + bb.1: + successors: %bb.2 + liveins: $vgpr0_vgpr1, $vgpr1_vgpr2, $vgpr3_vgpr4 + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + GLOBAL_STORE_DWORDX2 $vgpr1_vgpr2, $vgpr3_vgpr4, 0, 0, implicit $exec + S_BRANCH %bb.2 + + bb.2: + S_ENDPGM 0 +... +)"; + ASSERT_TRUE(parseMIR(MIR)); + MachineFunction &MF = getMF("SimpleDisjointMerge"); + MachineBasicBlock &BB0 = *MF.getBlockNumbered(0); + MachineBasicBlock &BB1 = *MF.getBlockNumbered(1); + MachineBasicBlock &BB2 = *MF.getBlockNumbered(2); + + visitAll(BB0); + visitAll(BB1); + auto &ET = visitAll(BB2); + + auto LoadCnt = TrackerRecordsChecker(ET, AMDGPU::LOAD_CNT); + EXPECT_EQ(LoadCnt.getCount(), 1u); + EXPECT_FALSE(LoadCnt.empty()); + EXPECT_EQ(LoadCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_LOAD_DWORD); + EXPECT_EQ(LoadCnt.cur().getAge(), 0u); + EXPECT_FALSE(LoadCnt.next()); + + auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); + EXPECT_EQ(DsCnt.getCount(), 1u); + EXPECT_FALSE(DsCnt.empty()); + EXPECT_EQ(DsCnt.cur().getMI()->getOpcode(), AMDGPU::DS_READ_B32_gfx9); + EXPECT_EQ(DsCnt.cur().getAge(), 0u); + EXPECT_FALSE(DsCnt.next()); + + auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); + EXPECT_TRUE(ExpCnt.unused()); + + auto StoreCnt = TrackerRecordsChecker(ET, AMDGPU::STORE_CNT); + EXPECT_EQ(StoreCnt.getCount(), 2u); + EXPECT_FALSE(StoreCnt.empty()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getAge(), 1u); + EXPECT_TRUE(StoreCnt.next()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_FALSE(StoreCnt.next()); +} + +/// Merge with divergent events in a counter. +TEST_F(AMDGPUGFX12EventTrackingTest, DivergenceMerge) { + StringRef MIR = R"( +name: DivergenceMerge +body: | + bb.0: + successors: %bb.2 + liveins: $vgpr1, $vgpr2 + + $vgpr0 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + S_BRANCH %bb.2 + + bb.1: + successors: %bb.2 + liveins: $vgpr1_vgpr2, $vgpr3_vgpr4, $vgpr4_vgpr5 + $vgpr4 = GLOBAL_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec + GLOBAL_STORE_DWORDX2 $vgpr1_vgpr2, $vgpr3_vgpr4, 0, 0, implicit $exec + S_BRANCH %bb.2 + + bb.2: + S_ENDPGM 0 +... +)"; + ASSERT_TRUE(parseMIR(MIR)); + MachineFunction &MF = getMF("DivergenceMerge"); + MachineBasicBlock &BB0 = *MF.getBlockNumbered(0); + MachineBasicBlock &BB1 = *MF.getBlockNumbered(1); + MachineBasicBlock &BB2 = *MF.getBlockNumbered(2); + + visitAll(BB0); + visitAll(BB1); + auto &ET = visitAll(BB2); + + auto LoadCnt = TrackerRecordsChecker(ET, AMDGPU::LOAD_CNT); + EXPECT_EQ(LoadCnt.getCount(), 1u); + EXPECT_FALSE(LoadCnt.empty()); + EXPECT_EQ(LoadCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_LOAD_DWORD); + EXPECT_EQ(LoadCnt.cur().getAge(), 0u); + EXPECT_FALSE(LoadCnt.next()); + + auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); + EXPECT_EQ(DsCnt.getCount(), 1u); + EXPECT_FALSE(DsCnt.empty()); + EXPECT_EQ(DsCnt.cur().getMI()->getOpcode(), AMDGPU::DS_READ_B32_gfx9); + EXPECT_EQ(DsCnt.cur().getAge(), 0u); + EXPECT_FALSE(DsCnt.next()); + + auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); + EXPECT_TRUE(ExpCnt.unused()); + + auto StoreCnt = TrackerRecordsChecker(ET, AMDGPU::STORE_CNT); + // Count is 1 because we have 1 event max across all predecessors. + EXPECT_EQ(StoreCnt.getCount(), 1u); + EXPECT_FALSE(StoreCnt.empty()); + // First successor has GLOBAL_STORE_DWORD at age 0 + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_TRUE(StoreCnt.next()); + // Second successor has GLOBAL_STORE_DWORDX2 at age 0 too + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_FALSE(StoreCnt.next()); +} + +/// Basic diamond CFG, the store is carried all the way into bb3 and uniqued +/// again so only 1 instance of the record is present in bb3. +TEST_F(AMDGPUGFX12EventTrackingTest, BasicDiamond) { + StringRef MIR = R"( +name: BasicDiamond +body: | + bb.0: + successors: %bb.1, %bb.2 + liveins: $vgpr0_vgpr1, $vgpr2, $vgpr4_vgpr5, $scc + + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + S_CBRANCH_SCC1 %bb.1, implicit $scc + S_BRANCH %bb.2 + + bb.1: + successors: %bb.3 + liveins: $vgpr1 + $vgpr0 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + liveins: $vgpr4_vgpr5 + $vgpr4 = GLOBAL_LOAD_DWORD $vgpr4_vgpr5, 0, 0, implicit $exec + S_BRANCH %bb.3 + + bb.3: + S_ENDPGM 0 +... +)"; + ASSERT_TRUE(parseMIR(MIR)); + MachineFunction &MF = getMF("BasicDiamond"); + MachineBasicBlock &BB0 = *MF.getBlockNumbered(0); + MachineBasicBlock &BB1 = *MF.getBlockNumbered(1); + MachineBasicBlock &BB2 = *MF.getBlockNumbered(2); + MachineBasicBlock &BB3 = *MF.getBlockNumbered(3); + + visitAll(BB0); + visitAll(BB1); + visitAll(BB2); + auto &ET = visitAll(BB3); + + auto LoadCnt = TrackerRecordsChecker(ET, AMDGPU::LOAD_CNT); + EXPECT_EQ(LoadCnt.getCount(), 1u); + EXPECT_FALSE(LoadCnt.empty()); + EXPECT_EQ(LoadCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_LOAD_DWORD); + EXPECT_EQ(LoadCnt.cur().getAge(), 0u); + EXPECT_FALSE(LoadCnt.next()); + + auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); + EXPECT_EQ(DsCnt.getCount(), 1u); + EXPECT_FALSE(DsCnt.empty()); + EXPECT_EQ(DsCnt.cur().getMI()->getOpcode(), AMDGPU::DS_READ_B32_gfx9); + EXPECT_EQ(DsCnt.cur().getAge(), 0u); + EXPECT_FALSE(DsCnt.next()); + + auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); + EXPECT_TRUE(ExpCnt.unused()); + + auto StoreCnt = TrackerRecordsChecker(ET, AMDGPU::STORE_CNT); + EXPECT_EQ(StoreCnt.getCount(), 1u); + EXPECT_FALSE(StoreCnt.empty()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_FALSE(StoreCnt.next()); +} + +/// Asymetrical diamond +/// - bb0 has two stores +/// - bb1 adds another store without any waits. +/// - bb2 adds a store and waits on the two stores from bb0 afterwards +/// +/// The timeline will be: +/// - Stores from bb0 exist at 1/2 +/// - The added store from bb1 exist at age 0 +/// - The added store from bb2 exist at age 0 +TEST_F(AMDGPUGFX12EventTrackingTest, AsymetricalDiamond) { + StringRef MIR = R"( +name: AsymetricalDiamond +body: | + bb.0: + successors: %bb.1, %bb.2 + liveins: $vgpr0_vgpr1, $vgpr1_vgpr2, $vgpr3_vgpr4, $scc + + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + GLOBAL_STORE_DWORDX2 $vgpr1_vgpr2, $vgpr3_vgpr4, 0, 0, implicit $exec + S_CBRANCH_SCC1 %bb.1, implicit $scc + S_BRANCH %bb.2 + + bb.1: + successors: %bb.3 + liveins: $vgpr1_vgpr2, $vgpr3 + GLOBAL_STORE_DWORD $vgpr1_vgpr2, $vgpr3, 0, 0, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + liveins: $vgpr0_vgpr1, $vgpr2 + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + S_BRANCH %bb.3 + + bb.3: + S_ENDPGM 0 +... +)"; + ASSERT_TRUE(parseMIR(MIR)); + MachineFunction &MF = getMF("AsymetricalDiamond"); + MachineBasicBlock &BB0 = *MF.getBlockNumbered(0); + MachineBasicBlock &BB1 = *MF.getBlockNumbered(1); + MachineBasicBlock &BB2 = *MF.getBlockNumbered(2); + MachineBasicBlock &BB3 = *MF.getBlockNumbered(3); + + visitAll(BB0); + visitAll(BB1); + visitAll(BB2, + /*AfterVisit=*/[&](EventTracker &ET) { ET.drain(STORE_CNT, 1); }); + auto &ET = visitAll(BB3); + + auto LoadCnt = TrackerRecordsChecker(ET, AMDGPU::LOAD_CNT); + EXPECT_TRUE(LoadCnt.unused()); + + auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); + EXPECT_TRUE(DsCnt.unused()); + + auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); + EXPECT_TRUE(ExpCnt.unused()); + + auto StoreCnt = TrackerRecordsChecker(ET, AMDGPU::STORE_CNT); + + EXPECT_EQ(StoreCnt.getCount(), 3u); + EXPECT_FALSE(StoreCnt.empty()); + + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getMI()->getParent(), &BB0); + EXPECT_EQ(StoreCnt.cur().getAge(), 2u); + EXPECT_TRUE(StoreCnt.next()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORDX2); + EXPECT_EQ(StoreCnt.cur().getAge(), 1u); + EXPECT_TRUE(StoreCnt.next()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getMI()->getParent(), &BB1); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_TRUE(StoreCnt.next()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getMI()->getParent(), &BB2); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_FALSE(StoreCnt.next()); +} + +/// Asymetrical diamond +/// - bb0 has one store +/// - bb1 just falls through +/// - bb2 adds 1 more store. +/// +/// The timeline will showcase how the value of a counter can be different from +/// the content of the timeline. +/// - The timeline will have all records at an age of 0. +/// - The counter will still be at 2. +TEST_F(AMDGPUGFX12EventTrackingTest, AsymetricalDiamond2) { + StringRef MIR = R"( +name: AsymetricalDiamond2 +body: | + bb.0: + successors: %bb.1, %bb.2 + liveins: $vgpr0_vgpr1, $vgpr2, $scc + + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + S_CBRANCH_SCC1 %bb.1, implicit $scc + S_BRANCH %bb.2 + + bb.1: + successors: %bb.3 + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + liveins: $vgpr0_vgpr1, $vgpr2 + GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec + S_BRANCH %bb.3 + + bb.3: + S_ENDPGM 0 +... +)"; + ASSERT_TRUE(parseMIR(MIR)); + MachineFunction &MF = getMF("AsymetricalDiamond2"); + MachineBasicBlock &BB0 = *MF.getBlockNumbered(0); + MachineBasicBlock &BB1 = *MF.getBlockNumbered(1); + MachineBasicBlock &BB2 = *MF.getBlockNumbered(2); + MachineBasicBlock &BB3 = *MF.getBlockNumbered(3); + + visitAll(BB0); + visitAll(BB1); + visitAll(BB2); + + auto &ET = visitAll(BB3); + + auto LoadCnt = TrackerRecordsChecker(ET, AMDGPU::LOAD_CNT); + EXPECT_TRUE(LoadCnt.unused()); + + auto DsCnt = TrackerRecordsChecker(ET, AMDGPU::DS_CNT); + EXPECT_TRUE(DsCnt.unused()); + + auto ExpCnt = TrackerRecordsChecker(ET, AMDGPU::EXP_CNT); + EXPECT_TRUE(ExpCnt.unused()); + + auto StoreCnt = TrackerRecordsChecker(ET, AMDGPU::STORE_CNT); + + EXPECT_EQ(StoreCnt.getCount(), 2u); + EXPECT_FALSE(StoreCnt.empty()); + + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getMI()->getParent(), &BB0); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_TRUE(StoreCnt.next()); + EXPECT_EQ(StoreCnt.cur().getMI()->getOpcode(), AMDGPU::GLOBAL_STORE_DWORD); + EXPECT_EQ(StoreCnt.cur().getMI()->getParent(), &BB2); + EXPECT_EQ(StoreCnt.cur().getAge(), 0u); + EXPECT_FALSE(StoreCnt.next()); +} + } // namespace _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
