From 619c2dbf46a1817ee6ead3a80662f11f839ded06 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Tue, 21 May 2024 12:55:07 -0700 Subject: [PATCH 1/7] [AMDGPU] NFC: Add BBLiveOutMap & LiveOut Cache Change-Id: I63cfd44e635cc4bee0e6780ca43b692c46e940b7 --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 56 ++++++++++++++++++++- llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 7 +++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 94d93390d0916..a4d05f62a7f74 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -58,6 +58,11 @@ static cl::opt "Wave Limited (amdgpu-limit-wave-threshold)."), cl::init(false)); +static cl::opt GCNTrackers( + "amdgpu-use-amdgpu-trackers", cl::Hidden, + cl::desc("Use the AMDGPU specific RPTrackers during scheduling"), + cl::init(false)); + const unsigned ScheduleMetrics::ScaleFactor = 100; GCNSchedStrategy::GCNSchedStrategy(const MachineSchedContext *C) @@ -526,6 +531,19 @@ GCNScheduleDAGMILive::getRealRegPressure(unsigned RegionIdx) const { return RPTracker.moveMaxPressure(); } +static MachineInstr *getLastMIForRegion(MachineBasicBlock::iterator RegionBegin, + MachineBasicBlock::iterator RegionEnd) { + MachineInstr *LastMI; + auto *BB = RegionBegin->getParent(); + if (RegionEnd != BB->end() && !RegionEnd->isDebugInstr()) + LastMI = &*RegionEnd; + else if (RegionEnd == BB->end()) + LastMI = &*prev_nodbg(RegionEnd, RegionBegin); + else + LastMI = &*skipDebugInstructionsBackward(RegionEnd, RegionBegin); + return LastMI; +} + void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx, const MachineBasicBlock *MBB) { GCNDownwardRPTracker RPTracker(*LIS); @@ -597,6 +615,16 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx, RPTracker.advanceBeforeNext(); MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs(); } + + if (GCNTrackers) { + assert(LiveOuts.size() == Regions.size()); + for (unsigned RegionIdx = 0; RegionIdx < Regions.size(); RegionIdx++) { + auto RegionBegin = Regions[RegionIdx].first; + auto RegionEnd = Regions[RegionIdx].second; + MachineInstr *LastMI = getLastMIForRegion(RegionBegin, RegionEnd); + LiveOuts[RegionIdx] = BBLiveOutMap.lookup(LastMI); + } + } } DenseMap @@ -616,11 +644,24 @@ GCNScheduleDAGMILive::getBBLiveInMap() const { return getLiveRegMap(BBStarters, false /*After*/, *LIS); } +DenseMap +GCNScheduleDAGMILive::getBBLiveOutMap() const { + assert(!Regions.empty()); + std::vector BBEnders; + BBEnders.reserve(Regions.size()); + auto I = Regions.rbegin(), E = Regions.rend(); + for (; I != E; I++) + BBEnders.push_back(getLastMIForRegion(I->first, I->second)); + + return getLiveRegMap(BBEnders, true /*After*/, *LIS); +} + void GCNScheduleDAGMILive::finalizeSchedule() { // Start actual scheduling here. This function is called by the base // MachineScheduler after all regions have been recorded by // GCNScheduleDAGMILive::schedule(). LiveIns.resize(Regions.size()); + LiveOuts.resize(Regions.size()); Pressure.resize(Regions.size()); RescheduleRegions.resize(Regions.size()); RegionsWithHighRP.resize(Regions.size()); @@ -639,8 +680,12 @@ void GCNScheduleDAGMILive::finalizeSchedule() { void GCNScheduleDAGMILive::runSchedStages() { LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n"); - if (!Regions.empty()) + if (!Regions.empty()) { BBLiveInMap = getBBLiveInMap(); + if (GCNTrackers) { + BBLiveOutMap = getBBLiveOutMap(); + } + } GCNSchedStrategy &S = static_cast(*SchedImpl); while (S.advanceStage()) { @@ -1499,6 +1544,15 @@ bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST, DAG.Regions = NewRegions; DAG.RescheduleRegions = NewRescheduleRegions; + if (GCNTrackers) { + DAG.BBLiveOutMap = DAG.getBBLiveOutMap(); + auto I = DAG.Regions.begin(), E = DAG.Regions.end(); + for (; I != E; I++) { + MachineInstr *LastMI = getLastMIForRegion(I->first, I->second); + DAG.LiveOuts.push_back(DAG.BBLiveOutMap.lookup(LastMI)); + } + } + SIMachineFunctionInfo &MFI = *MF.getInfo(); MFI.increaseOccupancy(MF, ++DAG.MinOccupancy); diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h index 2084aae4128ff..243bb7f0c094d 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h @@ -205,6 +205,9 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive { // Region live-in cache. SmallVector LiveIns; + // Region live-out cache. + SmallVector LiveOuts; + // Region pressure cache. SmallVector Pressure; @@ -215,6 +218,10 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive { DenseMap getBBLiveInMap() const; + DenseMap BBLiveOutMap; + + DenseMap getBBLiveOutMap() const; + // Return current region pressure. GCNRegPressure getRealRegPressure(unsigned RegionIdx) const; From 9f2f32bb361e46e088914b804fcb635c2970c66a Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Thu, 23 May 2024 11:03:27 -0700 Subject: [PATCH 2/7] Review Comments Change-Id: Iaeaa9bc5b037d78ab965c3bc1778d424e37eb546 --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index a4d05f62a7f74..6f792ce24350a 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -533,15 +533,10 @@ GCNScheduleDAGMILive::getRealRegPressure(unsigned RegionIdx) const { static MachineInstr *getLastMIForRegion(MachineBasicBlock::iterator RegionBegin, MachineBasicBlock::iterator RegionEnd) { - MachineInstr *LastMI; - auto *BB = RegionBegin->getParent(); - if (RegionEnd != BB->end() && !RegionEnd->isDebugInstr()) - LastMI = &*RegionEnd; - else if (RegionEnd == BB->end()) - LastMI = &*prev_nodbg(RegionEnd, RegionBegin); - else - LastMI = &*skipDebugInstructionsBackward(RegionEnd, RegionBegin); - return LastMI; + auto REnd = RegionEnd == RegionBegin->getParent()->end() + ? std::prev(RegionEnd) + : RegionEnd; + return &*skipDebugInstructionsBackward(REnd, RegionBegin); } void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx, @@ -641,7 +636,7 @@ GCNScheduleDAGMILive::getBBLiveInMap() const { ++I; } while (I != E && I->first->getParent() == BB); } while (I != E); - return getLiveRegMap(BBStarters, false /*After*/, *LIS); + return getLiveRegMap(BBStarters, /*After=*/false, *LIS); } DenseMap @@ -653,7 +648,7 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const { for (; I != E; I++) BBEnders.push_back(getLastMIForRegion(I->first, I->second)); - return getLiveRegMap(BBEnders, true /*After*/, *LIS); + return getLiveRegMap(BBEnders, /*After= */true, *LIS); } void GCNScheduleDAGMILive::finalizeSchedule() { From ea707692a8c618f7f80d815645143206a39f2dab Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Thu, 23 May 2024 11:11:50 -0700 Subject: [PATCH 3/7] Formatting Change-Id: I8418e9dd9571feb8cdbb32623f21ecb2ff41aa9e --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 6f792ce24350a..215fe79cfc728 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -648,7 +648,7 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const { for (; I != E; I++) BBEnders.push_back(getLastMIForRegion(I->first, I->second)); - return getLiveRegMap(BBEnders, /*After= */true, *LIS); + return getLiveRegMap(BBEnders, /*After= */ true, *LIS); } void GCNScheduleDAGMILive::finalizeSchedule() { From a22d7e09fe86b2186f7aa5988f0d4f9324a6d330 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Tue, 28 May 2024 11:13:32 -0700 Subject: [PATCH 4/7] Add wrapper for BBLiveRegMap Change-Id: I4f8fa1415194aeda4cc6318c6213e9c71a2d50dc --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 46 ++++++++++----------- llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 37 ++++++++++++++--- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 215fe79cfc728..7cd009f00a9b7 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -489,7 +489,8 @@ GCNScheduleDAGMILive::GCNScheduleDAGMILive( MachineSchedContext *C, std::unique_ptr S) : ScheduleDAGMILive(C, std::move(S)), ST(MF.getSubtarget()), MFI(*MF.getInfo()), - StartingOccupancy(MFI.getOccupancy()), MinOccupancy(StartingOccupancy) { + StartingOccupancy(MFI.getOccupancy()), MinOccupancy(StartingOccupancy), + RegionLiveOuts(this, /*IsLiveOut=*/true) { LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n"); if (RelaxedOcc) { @@ -610,16 +611,6 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx, RPTracker.advanceBeforeNext(); MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs(); } - - if (GCNTrackers) { - assert(LiveOuts.size() == Regions.size()); - for (unsigned RegionIdx = 0; RegionIdx < Regions.size(); RegionIdx++) { - auto RegionBegin = Regions[RegionIdx].first; - auto RegionEnd = Regions[RegionIdx].second; - MachineInstr *LastMI = getLastMIForRegion(RegionBegin, RegionEnd); - LiveOuts[RegionIdx] = BBLiveOutMap.lookup(LastMI); - } - } } DenseMap @@ -651,12 +642,29 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const { return getLiveRegMap(BBEnders, /*After= */ true, *LIS); } +void RegionPressureMap::buildLiveRegMap() { + if (IsMapGenerated) { + IdxToInstruction.clear(); + BBLiveRegMap.clear(); + IsMapGenerated = false; + } + + BBLiveRegMap = IsLiveOut ? DAG->getBBLiveOutMap() : DAG->getBBLiveInMap(); + for (unsigned I = 0; I < DAG->Regions.size(); I++) { + MachineInstr *RegionKey; + RegionKey = IsLiveOut ? getLastMIForRegion(DAG->Regions[I].first, + DAG->Regions[I].second) + : &*DAG->Regions[I].first; + IdxToInstruction[I] = RegionKey; + } + IsMapGenerated = true; +} + void GCNScheduleDAGMILive::finalizeSchedule() { // Start actual scheduling here. This function is called by the base // MachineScheduler after all regions have been recorded by // GCNScheduleDAGMILive::schedule(). LiveIns.resize(Regions.size()); - LiveOuts.resize(Regions.size()); Pressure.resize(Regions.size()); RescheduleRegions.resize(Regions.size()); RegionsWithHighRP.resize(Regions.size()); @@ -677,9 +685,8 @@ void GCNScheduleDAGMILive::runSchedStages() { if (!Regions.empty()) { BBLiveInMap = getBBLiveInMap(); - if (GCNTrackers) { - BBLiveOutMap = getBBLiveOutMap(); - } + if (GCNTrackers) + RegionLiveOuts.buildLiveRegMap(); } GCNSchedStrategy &S = static_cast(*SchedImpl); @@ -1539,15 +1546,6 @@ bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST, DAG.Regions = NewRegions; DAG.RescheduleRegions = NewRescheduleRegions; - if (GCNTrackers) { - DAG.BBLiveOutMap = DAG.getBBLiveOutMap(); - auto I = DAG.Regions.begin(), E = DAG.Regions.end(); - for (; I != E; I++) { - MachineInstr *LastMI = getLastMIForRegion(I->first, I->second); - DAG.LiveOuts.push_back(DAG.BBLiveOutMap.lookup(LastMI)); - } - } - SIMachineFunctionInfo &MFI = *MF.getInfo(); MFI.increaseOccupancy(MF, ++DAG.MinOccupancy); diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h index 243bb7f0c094d..9001b16ec256d 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h @@ -163,6 +163,35 @@ inline raw_ostream &operator<<(raw_ostream &OS, const ScheduleMetrics &Sm) { return OS; } +class GCNScheduleDAGMILive; +class RegionPressureMap { + GCNScheduleDAGMILive *DAG; + // The live in/out pressure as indexed by the first or last MI in the region + // before scheduling. + DenseMap BBLiveRegMap; + // The mapping of RegionIDx to key instruction + DenseMap IdxToInstruction; + // Whether we are calculating LiveOuts or LiveIns + bool IsLiveOut; + // Whether or not the maps have been generated + bool IsMapGenerated = false; + +public: + RegionPressureMap() {} + RegionPressureMap(GCNScheduleDAGMILive *GCNDAG, bool LiveOut) + : DAG(GCNDAG), IsLiveOut(LiveOut) {} + // Build the Instr->LiveReg and RegionIdx->Instr maps + void buildLiveRegMap(); + + // Retrieve the LiveReg for a given RegionIdx + GCNRPTracker::LiveRegSet &getLiveRegsForRegionIdx(unsigned RegionIdx) { + assert(IsMapGenerated); + assert(IdxToInstruction.find(RegionIdx) != IdxToInstruction.end()); + MachineInstr *Key = IdxToInstruction[RegionIdx]; + return BBLiveRegMap[Key]; + } +}; + class GCNScheduleDAGMILive final : public ScheduleDAGMILive { friend class GCNSchedStage; friend class OccInitialScheduleStage; @@ -170,6 +199,7 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive { friend class ClusteredLowOccStage; friend class PreRARematStage; friend class ILPInitialScheduleStage; + friend class RegionPressureMap; const GCNSubtarget &ST; @@ -205,9 +235,6 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive { // Region live-in cache. SmallVector LiveIns; - // Region live-out cache. - SmallVector LiveOuts; - // Region pressure cache. SmallVector Pressure; @@ -218,10 +245,10 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive { DenseMap getBBLiveInMap() const; - DenseMap BBLiveOutMap; - DenseMap getBBLiveOutMap() const; + RegionPressureMap RegionLiveOuts; + // Return current region pressure. GCNRegPressure getRealRegPressure(unsigned RegionIdx) const; From 18cf1b9cce0d4c8d727e525a1e43963486f98815 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Fri, 14 Jun 2024 08:21:24 -0700 Subject: [PATCH 5/7] Review comments Change-Id: Ic703ad43fa7fd49afcb1d17c30325eb0dd7a8355 --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 21 ++++++++------------- llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 3 --- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 7cd009f00a9b7..f770958c6c0d1 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -635,29 +635,24 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const { assert(!Regions.empty()); std::vector BBEnders; BBEnders.reserve(Regions.size()); - auto I = Regions.rbegin(), E = Regions.rend(); - for (; I != E; I++) - BBEnders.push_back(getLastMIForRegion(I->first, I->second)); + for (auto &[RegionBegin, RegionEnd] : reverse(Regions)) + BBEnders.push_back(getLastMIForRegion(RegionBegin, RegionEnd)); return getLiveRegMap(BBEnders, /*After= */ true, *LIS); } void RegionPressureMap::buildLiveRegMap() { - if (IsMapGenerated) { - IdxToInstruction.clear(); - BBLiveRegMap.clear(); - IsMapGenerated = false; - } + IdxToInstruction.clear(); + BBLiveRegMap.clear(); BBLiveRegMap = IsLiveOut ? DAG->getBBLiveOutMap() : DAG->getBBLiveInMap(); for (unsigned I = 0; I < DAG->Regions.size(); I++) { - MachineInstr *RegionKey; - RegionKey = IsLiveOut ? getLastMIForRegion(DAG->Regions[I].first, - DAG->Regions[I].second) - : &*DAG->Regions[I].first; + MachineInstr *RegionKey = + IsLiveOut + ? getLastMIForRegion(DAG->Regions[I].first, DAG->Regions[I].second) + : &*DAG->Regions[I].first; IdxToInstruction[I] = RegionKey; } - IsMapGenerated = true; } void GCNScheduleDAGMILive::finalizeSchedule() { diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h index 9001b16ec256d..e7e086ffe513c 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h @@ -173,8 +173,6 @@ class RegionPressureMap { DenseMap IdxToInstruction; // Whether we are calculating LiveOuts or LiveIns bool IsLiveOut; - // Whether or not the maps have been generated - bool IsMapGenerated = false; public: RegionPressureMap() {} @@ -185,7 +183,6 @@ class RegionPressureMap { // Retrieve the LiveReg for a given RegionIdx GCNRPTracker::LiveRegSet &getLiveRegsForRegionIdx(unsigned RegionIdx) { - assert(IsMapGenerated); assert(IdxToInstruction.find(RegionIdx) != IdxToInstruction.end()); MachineInstr *Key = IdxToInstruction[RegionIdx]; return BBLiveRegMap[Key]; From 2fad27e1302452d67d5948af0422b0b94a479509 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Fri, 14 Jun 2024 14:03:45 -0700 Subject: [PATCH 6/7] Remove clear Change-Id: I90da0419eec73e7166e7b156796462a8ce6d5497 --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index f770958c6c0d1..ffbbeb02a0f5a 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -643,7 +643,6 @@ GCNScheduleDAGMILive::getBBLiveOutMap() const { void RegionPressureMap::buildLiveRegMap() { IdxToInstruction.clear(); - BBLiveRegMap.clear(); BBLiveRegMap = IsLiveOut ? DAG->getBBLiveOutMap() : DAG->getBBLiveInMap(); for (unsigned I = 0; I < DAG->Regions.size(); I++) { From d4c1c698ff9f33446b8d7048be1016e178bef526 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Fri, 14 Jun 2024 14:03:45 -0700 Subject: [PATCH 7/7] Address Comments Change-Id: I90da0419eec73e7166e7b156796462a8ce6d5497 --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 25 +++++++++++---------- llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 13 +++++++++-- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index ffbbeb02a0f5a..db8dd45ec0441 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -614,37 +614,38 @@ void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx, } DenseMap -GCNScheduleDAGMILive::getBBLiveInMap() const { +GCNScheduleDAGMILive::getRegionLiveInMap() const { assert(!Regions.empty()); - std::vector BBStarters; - BBStarters.reserve(Regions.size()); + std::vector RegionFirstMIs; + RegionFirstMIs.reserve(Regions.size()); auto I = Regions.rbegin(), E = Regions.rend(); auto *BB = I->first->getParent(); do { auto *MI = &*skipDebugInstructionsForward(I->first, I->second); - BBStarters.push_back(MI); + RegionFirstMIs.push_back(MI); do { ++I; } while (I != E && I->first->getParent() == BB); } while (I != E); - return getLiveRegMap(BBStarters, /*After=*/false, *LIS); + return getLiveRegMap(RegionFirstMIs, /*After=*/false, *LIS); } DenseMap -GCNScheduleDAGMILive::getBBLiveOutMap() const { +GCNScheduleDAGMILive::getRegionLiveOutMap() const { assert(!Regions.empty()); - std::vector BBEnders; - BBEnders.reserve(Regions.size()); + std::vector RegionLastMIs; + RegionLastMIs.reserve(Regions.size()); for (auto &[RegionBegin, RegionEnd] : reverse(Regions)) - BBEnders.push_back(getLastMIForRegion(RegionBegin, RegionEnd)); + RegionLastMIs.push_back(getLastMIForRegion(RegionBegin, RegionEnd)); - return getLiveRegMap(BBEnders, /*After= */ true, *LIS); + return getLiveRegMap(RegionLastMIs, /*After=*/true, *LIS); } void RegionPressureMap::buildLiveRegMap() { IdxToInstruction.clear(); - BBLiveRegMap = IsLiveOut ? DAG->getBBLiveOutMap() : DAG->getBBLiveInMap(); + BBLiveRegMap = + IsLiveOut ? DAG->getRegionLiveOutMap() : DAG->getRegionLiveInMap(); for (unsigned I = 0; I < DAG->Regions.size(); I++) { MachineInstr *RegionKey = IsLiveOut @@ -678,7 +679,7 @@ void GCNScheduleDAGMILive::runSchedStages() { LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n"); if (!Regions.empty()) { - BBLiveInMap = getBBLiveInMap(); + BBLiveInMap = getRegionLiveInMap(); if (GCNTrackers) RegionLiveOuts.buildLiveRegMap(); } diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h index e7e086ffe513c..a71f3fd0dd469 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h @@ -238,12 +238,21 @@ class GCNScheduleDAGMILive final : public ScheduleDAGMILive { // Temporary basic block live-in cache. DenseMap MBBLiveIns; + // The map of the initial first region instruction to region live in registers DenseMap BBLiveInMap; - DenseMap getBBLiveInMap() const; + // Calculate the map of the initial first region instruction to region live in + // registers + DenseMap getRegionLiveInMap() const; - DenseMap getBBLiveOutMap() const; + // Calculate the map of the initial last region instruction to region live out + // registers + DenseMap + getRegionLiveOutMap() const; + // The live out registers per region. These are internally stored as a map of + // the initial last region instruction to region live out registers, but can + // be retreived with the regionIdx by calls to getLiveRegsForRegionIdx. RegionPressureMap RegionLiveOuts; // Return current region pressure.