Save LaneMask with livein registers
With subregister liveness enabled we can detect the case where only
parts of a register are live in, this is expressed as a 32bit lanemask.
The current code only keeps registers in the live-in list and therefore
enumerated all subregisters affected by the lanemask. This turned out to
be too conservative as the subregister may also cover additional parts
of the lanemask which are not live. Expressing a given lanemask by
enumerating a minimum set of subregisters is computationally expensive
so the best solution is to simply change the live-in list to store the
lanemasks as well. This will reduce memory usage for targets using
subregister liveness and slightly increase it for other targets
Differential Revision: http://reviews.llvm.org/D12442
llvm-svn: 247171
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index 7ba6768..bba8f51 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -149,8 +149,8 @@
// Examine the live-in regs of all successors.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
- for (unsigned LI : (*SI)->liveins()) {
- for (MCRegAliasIterator AI(LI, TRI, true); AI.isValid(); ++AI) {
+ for (const auto &LI : (*SI)->liveins()) {
+ for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI) {
unsigned Reg = *AI;
State->UnionGroups(Reg, 0);
KillIndices[Reg] = BB->size();
diff --git a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
index d8401ca..07e5342 100644
--- a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
@@ -57,8 +57,8 @@
// Examine the live-in regs of all successors.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
- for (unsigned LI : (*SI)->liveins()) {
- for (MCRegAliasIterator AI(LI, TRI, true); AI.isValid(); ++AI) {
+ for (const auto &LI : (*SI)->liveins()) {
+ for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI) {
unsigned Reg = *AI;
Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
KillIndices[Reg] = BBSize;
diff --git a/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp b/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
index 8bb5482..b11b497 100644
--- a/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
+++ b/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
@@ -110,8 +110,8 @@
// block.
for (MachineBasicBlock::succ_iterator S = MBB.succ_begin(),
E = MBB.succ_end(); S != E; S++)
- for (unsigned LI : (*S)->liveins())
- LivePhysRegs.set(LI);
+ for (const auto &LI : (*S)->liveins())
+ LivePhysRegs.set(LI.PhysReg);
// Now scan the instructions and delete dead ones, tracking physreg
// liveness as we go.
diff --git a/llvm/lib/CodeGen/ExecutionDepsFix.cpp b/llvm/lib/CodeGen/ExecutionDepsFix.cpp
index e43e95f..3eee1e2 100644
--- a/llvm/lib/CodeGen/ExecutionDepsFix.cpp
+++ b/llvm/lib/CodeGen/ExecutionDepsFix.cpp
@@ -375,8 +375,8 @@
// This is the entry block.
if (MBB->pred_empty()) {
- for (unsigned LI : MBB->liveins()) {
- for (int rx : regIndices(LI)) {
+ for (const auto &LI : MBB->liveins()) {
+ for (int rx : regIndices(LI.PhysReg)) {
// Treat function live-ins as if they were defined just before the first
// instruction. Usually, function arguments are set up immediately
// before the call.
diff --git a/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp b/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
index 9f2b485..53a2412 100644
--- a/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -305,8 +305,8 @@
// Create phi-defs at Begin for all live-in registers.
SlotIndex Begin = Indexes->getMBBStartIdx(MBB);
DEBUG(dbgs() << Begin << "\tBB#" << MBB->getNumber());
- for (unsigned LI : MBB->liveins()) {
- for (MCRegUnitIterator Units(LI, TRI); Units.isValid(); ++Units) {
+ for (const auto &LI : MBB->liveins()) {
+ for (MCRegUnitIterator Units(LI.PhysReg, TRI); Units.isValid(); ++Units) {
unsigned Unit = *Units;
LiveRange *LR = RegUnitRanges[Unit];
if (!LR) {
diff --git a/llvm/lib/CodeGen/LivePhysRegs.cpp b/llvm/lib/CodeGen/LivePhysRegs.cpp
index 30addff..61226b7 100644
--- a/llvm/lib/CodeGen/LivePhysRegs.cpp
+++ b/llvm/lib/CodeGen/LivePhysRegs.cpp
@@ -128,8 +128,8 @@
/// Add live-in registers of basic block \p MBB to \p LiveRegs.
static void addLiveIns(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB) {
- for (unsigned LI : MBB.liveins())
- LiveRegs.addReg(LI);
+ for (const auto &LI : MBB.liveins())
+ LiveRegs.addReg(LI.PhysReg);
}
/// Add pristine registers to the given \p LiveRegs. This function removes
diff --git a/llvm/lib/CodeGen/LiveVariables.cpp b/llvm/lib/CodeGen/LiveVariables.cpp
index 6e843b0..c4c4245 100644
--- a/llvm/lib/CodeGen/LiveVariables.cpp
+++ b/llvm/lib/CodeGen/LiveVariables.cpp
@@ -559,10 +559,10 @@
void LiveVariables::runOnBlock(MachineBasicBlock *MBB, const unsigned NumRegs) {
// Mark live-in registers as live-in.
SmallVector<unsigned, 4> Defs;
- for (unsigned LI : MBB->liveins()) {
- assert(TargetRegisterInfo::isPhysicalRegister(LI) &&
+ for (const auto &LI : MBB->liveins()) {
+ assert(TargetRegisterInfo::isPhysicalRegister(LI.PhysReg) &&
"Cannot have a live-in virtual register!");
- HandlePhysRegDef(LI, nullptr, Defs);
+ HandlePhysRegDef(LI.PhysReg, nullptr, Defs);
}
// Loop over all of the instructions, processing them.
@@ -600,10 +600,10 @@
MachineBasicBlock *SuccMBB = *SI;
if (SuccMBB->isEHPad())
continue;
- for (unsigned LI : SuccMBB->liveins()) {
- if (!TRI->isInAllocatableClass(LI))
+ for (const auto &LI : SuccMBB->liveins()) {
+ if (!TRI->isInAllocatableClass(LI.PhysReg))
// Ignore other live-ins, e.g. those that are live into landing pads.
- LiveOuts.insert(LI);
+ LiveOuts.insert(LI.PhysReg);
}
}
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index bfcc56f..9081e66 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -28,6 +28,7 @@
#include "llvm/IR/Module.h"
#include "llvm/IR/ModuleSlotTracker.h"
#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Format.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/YAMLTraits.h"
@@ -474,11 +475,13 @@
if (!MBB.livein_empty()) {
OS.indent(2) << "liveins: ";
bool First = true;
- for (unsigned LI : MBB.liveins()) {
+ for (const auto &LI : MBB.liveins()) {
if (!First)
OS << ", ";
First = false;
- printReg(LI, OS, TRI);
+ printReg(LI.PhysReg, OS, TRI);
+ if (LI.LaneMask != ~0u)
+ OS << format(":%08X", LI.LaneMask);
}
OS << "\n";
HasLineAttributes = true;
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index b2bbe09..025c0af 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -28,6 +28,7 @@
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetMachine.h"
@@ -277,8 +278,10 @@
if (!livein_empty()) {
if (Indexes) OS << '\t';
OS << " Live Ins:";
- for (unsigned LI : make_range(livein_begin(), livein_end())) {
- OS << ' ' << PrintReg(LI, TRI);
+ for (const auto &LI : make_range(livein_begin(), livein_end())) {
+ OS << ' ' << PrintReg(LI.PhysReg, TRI);
+ if (LI.LaneMask != ~0u)
+ OS << format(":%08X", LI.LaneMask);
}
OS << '\n';
}
@@ -321,15 +324,43 @@
OS << "BB#" << getNumber();
}
-void MachineBasicBlock::removeLiveIn(MCPhysReg Reg) {
- LiveInVector::iterator I = std::find(LiveIns.begin(), LiveIns.end(), Reg);
- if (I != LiveIns.end())
+void MachineBasicBlock::removeLiveIn(MCPhysReg Reg, unsigned LaneMask) {
+ LiveInVector::iterator I = std::find_if(
+ LiveIns.begin(), LiveIns.end(),
+ [Reg] (const RegisterMaskPair &LI) { return LI.PhysReg == Reg; });
+ if (I == LiveIns.end())
+ return;
+
+ I->LaneMask &= ~LaneMask;
+ if (I->LaneMask == 0)
LiveIns.erase(I);
}
-bool MachineBasicBlock::isLiveIn(MCPhysReg Reg) const {
- livein_iterator I = std::find(livein_begin(), livein_end(), Reg);
- return I != livein_end();
+bool MachineBasicBlock::isLiveIn(MCPhysReg Reg, unsigned LaneMask) const {
+ livein_iterator I = std::find_if(
+ LiveIns.begin(), LiveIns.end(),
+ [Reg] (const RegisterMaskPair &LI) { return LI.PhysReg == Reg; });
+ return I != livein_end() && (I->LaneMask & LaneMask) != 0;
+}
+
+void MachineBasicBlock::sortUniqueLiveIns() {
+ std::sort(LiveIns.begin(), LiveIns.end(),
+ [](const RegisterMaskPair &LI0, const RegisterMaskPair &LI1) {
+ return LI0.PhysReg < LI1.PhysReg;
+ });
+ // Liveins are sorted by physreg now we can merge their lanemasks.
+ LiveInVector::const_iterator I = LiveIns.begin();
+ LiveInVector::const_iterator J;
+ LiveInVector::iterator Out = LiveIns.begin();
+ for (; I != LiveIns.end(); ++Out, I = J) {
+ unsigned PhysReg = I->PhysReg;
+ unsigned LaneMask = I->LaneMask;
+ for (J = std::next(I); J != LiveIns.end() && J->PhysReg == PhysReg; ++J)
+ LaneMask |= J->LaneMask;
+ Out->PhysReg = PhysReg;
+ Out->LaneMask = LaneMask;
+ }
+ LiveIns.erase(Out, LiveIns.end());
}
unsigned
@@ -803,7 +834,7 @@
i->getOperand(ni+1).setMBB(NMBB);
// Inherit live-ins from the successor
- for (unsigned LI : Succ->liveins())
+ for (const auto &LI : Succ->liveins())
NMBB->addLiveIn(LI);
// Update LiveVariables.
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index d2de7fe..3ae5aa5 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -534,8 +534,8 @@
// Conservatively treat live-in's as an external def.
// FIXME: That means a reload that're reused in successor block(s) will not
// be LICM'ed.
- for (unsigned LI : BB->liveins()) {
- for (MCRegAliasIterator AI(LI, TRI, true); AI.isValid(); ++AI)
+ for (const auto &LI : BB->liveins()) {
+ for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI)
PhysRegDefs.set(*AI);
}
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index f131d99..ef499a3 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -524,8 +524,8 @@
if (MRI->isSSA()) {
// If this block has allocatable physical registers live-in, check that
// it is an entry block or landing pad.
- for (unsigned LI : MBB->liveins()) {
- if (isAllocatable(LI) && !MBB->isEHPad() &&
+ for (const auto &LI : MBB->liveins()) {
+ if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
MBB != MBB->getParent()->begin()) {
report("MBB has allocable live-in, but isn't entry or landing-pad.", MBB);
}
@@ -694,12 +694,12 @@
}
regsLive.clear();
- for (unsigned LI : MBB->liveins()) {
- if (!TargetRegisterInfo::isPhysicalRegister(LI)) {
+ for (const auto &LI : MBB->liveins()) {
+ if (!TargetRegisterInfo::isPhysicalRegister(LI.PhysReg)) {
report("MBB live-in list contains non-physical register", MBB);
continue;
}
- for (MCSubRegIterator SubRegs(LI, TRI, /*IncludeSelf=*/true);
+ for (MCSubRegIterator SubRegs(LI.PhysReg, TRI, /*IncludeSelf=*/true);
SubRegs.isValid(); ++SubRegs)
regsLive.insert(*SubRegs);
}
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index 2ccee62..f4c076f 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -799,9 +799,9 @@
MachineBasicBlock::iterator MII = MBB->begin();
// Add live-in registers as live.
- for (unsigned LI : MBB->liveins())
- if (MRI->isAllocatable(LI))
- definePhysReg(MII, LI, regReserved);
+ for (const auto &LI : MBB->liveins())
+ if (MRI->isAllocatable(LI.PhysReg))
+ definePhysReg(MII, LI.PhysReg, regReserved);
SmallVector<unsigned, 8> VirtDead;
SmallVector<MachineInstr*, 32> Coalesced;
diff --git a/llvm/lib/CodeGen/RegisterScavenging.cpp b/llvm/lib/CodeGen/RegisterScavenging.cpp
index 471cdcd..e8c7439 100644
--- a/llvm/lib/CodeGen/RegisterScavenging.cpp
+++ b/llvm/lib/CodeGen/RegisterScavenging.cpp
@@ -31,9 +31,12 @@
#define DEBUG_TYPE "reg-scavenging"
/// setUsed - Set the register units of this register as used.
-void RegScavenger::setRegUsed(unsigned Reg) {
- for (MCRegUnitIterator RUI(Reg, TRI); RUI.isValid(); ++RUI)
- RegUnitsAvailable.reset(*RUI);
+void RegScavenger::setRegUsed(unsigned Reg, unsigned LaneMask) {
+ for (MCRegUnitMaskIterator RUI(Reg, TRI); RUI.isValid(); ++RUI) {
+ unsigned UnitMask = (*RUI).second;
+ if (UnitMask == 0 || (LaneMask & UnitMask) != 0)
+ RegUnitsAvailable.reset((*RUI).first);
+ }
}
void RegScavenger::initRegState() {
@@ -50,8 +53,8 @@
return;
// Live-in registers are in use.
- for (unsigned LI : MBB->liveins())
- setRegUsed(LI);
+ for (const auto &LI : MBB->liveins())
+ setRegUsed(LI.PhysReg, LI.LaneMask);
// Pristine CSRs are also unavailable.
const MachineFunction &MF = *MBB->getParent();
diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index 3123446..39cad7f 100644
--- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -242,9 +242,9 @@
assert(Uses.empty() && "Uses in set before adding deps?");
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
- for (unsigned LI : (*SI)->liveins()) {
- if (!Uses.contains(LI))
- Uses.insert(PhysRegSUOper(&ExitSU, -1, LI));
+ for (const auto &LI : (*SI)->liveins()) {
+ if (!Uses.contains(LI.PhysReg))
+ Uses.insert(PhysRegSUOper(&ExitSU, -1, LI.PhysReg));
}
}
}
@@ -1078,9 +1078,9 @@
// Examine the live-in regs of all successors.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI) {
- for (unsigned LI : (*SI)->liveins()) {
+ for (const auto &LI : (*SI)->liveins()) {
// Repeat, for reg and all subregs.
- for (MCSubRegIterator SubRegs(LI, TRI, /*IncludeSelf=*/true);
+ for (MCSubRegIterator SubRegs(LI.PhysReg, TRI, /*IncludeSelf=*/true);
SubRegs.isValid(); ++SubRegs)
LiveRegs.set(*SubRegs);
}
diff --git a/llvm/lib/CodeGen/TailDuplication.cpp b/llvm/lib/CodeGen/TailDuplication.cpp
index ca929d9..4a71c66 100644
--- a/llvm/lib/CodeGen/TailDuplication.cpp
+++ b/llvm/lib/CodeGen/TailDuplication.cpp
@@ -791,8 +791,8 @@
RS->enterBasicBlock(PredBB);
if (!PredBB->empty())
RS->forward(std::prev(PredBB->end()));
- for (unsigned LI : TailBB->liveins()) {
- if (!RS->isRegUsed(LI, false))
+ for (const auto &LI : TailBB->liveins()) {
+ if (!RS->isRegUsed(LI.PhysReg, false))
// If a register is previously livein to the tail but it's not live
// at the end of predecessor BB, then it should be added to its
// livein list.
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index fdc135f..ba1f571 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -277,13 +277,7 @@
if (LaneMask == 0)
continue;
MachineBasicBlock *MBB = MBBI->second;
- for (MCSubRegIndexIterator SR(PhysReg, TRI); SR.isValid(); ++SR) {
- unsigned SubReg = SR.getSubReg();
- unsigned SubRegIndex = SR.getSubRegIndex();
- unsigned SubRegLaneMask = TRI->getSubRegIndexLaneMask(SubRegIndex);
- if ((SubRegLaneMask & LaneMask) != 0)
- MBB->addLiveIn(SubReg);
- }
+ MBB->addLiveIn(PhysReg, LaneMask);
}
}