Switch spill weights from a basic loop depth estimation to BlockFrequencyInfo.
The main advantages here are way better heuristics, taking into account not
just loop depth but also __builtin_expect and other static heuristics and will
eventually learn how to use profile info. Most of the work in this patch is
pushing the MachineBlockFrequencyInfo analysis into the right places.
This is good for a 5% speedup on zlib's deflate (x86_64), there were some very
unfortunate spilling decisions in its hottest loop in longest_match(). Other
benchmarks I tried were mostly neutral.
This changes register allocation in subtle ways, update the tests for it.
2012-02-20-MachineCPBug.ll was deleted as it's very fragile and the instruction
it looked for was gone already (but the FileCheck pattern picked up unrelated
stuff).
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@184105 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp
index 38ae17d..b03c325 100644
--- a/lib/CodeGen/CalcSpillWeights.cpp
+++ b/lib/CodeGen/CalcSpillWeights.cpp
@@ -12,6 +12,7 @@
#include "llvm/ADT/SmallSet.h"
#include "llvm/CodeGen/CalcSpillWeights.h"
#include "llvm/CodeGen/LiveIntervalAnalysis.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -33,6 +34,7 @@
void CalculateSpillWeights::getAnalysisUsage(AnalysisUsage &au) const {
au.addRequired<LiveIntervals>();
+ au.addRequired<MachineBlockFrequencyInfo>();
au.addRequired<MachineLoopInfo>();
au.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(au);
@@ -45,7 +47,8 @@
LiveIntervals &LIS = getAnalysis<LiveIntervals>();
MachineRegisterInfo &MRI = MF.getRegInfo();
- VirtRegAuxInfo VRAI(MF, LIS, getAnalysis<MachineLoopInfo>());
+ VirtRegAuxInfo VRAI(MF, LIS, getAnalysis<MachineLoopInfo>(),
+ getAnalysis<MachineBlockFrequencyInfo>());
for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
if (MRI.reg_nodbg_empty(Reg))
@@ -107,12 +110,12 @@
return true;
}
-void VirtRegAuxInfo::CalculateWeightAndHint(LiveInterval &li) {
+void
+VirtRegAuxInfo::CalculateWeightAndHint(LiveInterval &li) {
MachineRegisterInfo &mri = MF.getRegInfo();
const TargetRegisterInfo &tri = *MF.getTarget().getRegisterInfo();
MachineBasicBlock *mbb = 0;
MachineLoop *loop = 0;
- unsigned loopDepth = 0;
bool isExiting = false;
float totalWeight = 0;
SmallPtrSet<MachineInstr*, 8> visited;
@@ -140,14 +143,14 @@
if (mi->getParent() != mbb) {
mbb = mi->getParent();
loop = Loops.getLoopFor(mbb);
- loopDepth = loop ? loop->getLoopDepth() : 0;
isExiting = loop ? loop->isLoopExiting(mbb) : false;
}
// Calculate instr weight.
bool reads, writes;
tie(reads, writes) = mi->readsWritesVirtualRegister(li.reg);
- weight = LiveIntervals::getSpillWeight(writes, reads, loopDepth);
+ weight = LiveIntervals::getSpillWeight(
+ writes, reads, MBFI.getBlockFreq(mi->getParent()));
// Give extra weight to what looks like a loop induction variable update.
if (writes && isExiting && LIS.isLiveOutOfMBB(li, mbb))
diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp
index dd00d77..db63b52 100644
--- a/lib/CodeGen/InlineSpiller.cpp
+++ b/lib/CodeGen/InlineSpiller.cpp
@@ -22,6 +22,7 @@
#include "llvm/CodeGen/LiveRangeEdit.h"
#include "llvm/CodeGen/LiveStackAnalysis.h"
#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -65,6 +66,7 @@
MachineRegisterInfo &MRI;
const TargetInstrInfo &TII;
const TargetRegisterInfo &TRI;
+ const MachineBlockFrequencyInfo &MBFI;
// Variables that are valid during spill(), but used by multiple methods.
LiveRangeEdit *Edit;
@@ -148,7 +150,8 @@
MFI(*mf.getFrameInfo()),
MRI(mf.getRegInfo()),
TII(*mf.getTarget().getInstrInfo()),
- TRI(*mf.getTarget().getRegisterInfo()) {}
+ TRI(*mf.getTarget().getRegisterInfo()),
+ MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()) {}
void spill(LiveRangeEdit &);
@@ -1290,5 +1293,5 @@
if (!RegsToSpill.empty())
spillAll();
- Edit->calculateRegClassAndHint(MF, Loops);
+ Edit->calculateRegClassAndHint(MF, Loops, MBFI);
}
diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp
index 1ca2d46..18eac4c 100644
--- a/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -28,6 +28,7 @@
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/IR/Value.h"
+#include "llvm/Support/BlockFrequency.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
@@ -605,21 +606,9 @@
}
float
-LiveIntervals::getSpillWeight(bool isDef, bool isUse, unsigned loopDepth) {
- // Limit the loop depth ridiculousness.
- if (loopDepth > 200)
- loopDepth = 200;
-
- // The loop depth is used to roughly estimate the number of times the
- // instruction is executed. Something like 10^d is simple, but will quickly
- // overflow a float. This expression behaves like 10^d for small d, but is
- // more tempered for large d. At d=200 we get 6.7e33 which leaves a bit of
- // headroom before overflow.
- // By the way, powf() might be unavailable here. For consistency,
- // We may take pow(double,double).
- float lc = std::pow(1 + (100.0 / (loopDepth + 10)), (double)loopDepth);
-
- return (isDef + isUse) * lc;
+LiveIntervals::getSpillWeight(bool isDef, bool isUse, BlockFrequency freq) {
+ const float Scale = 1.0f / BlockFrequency::getEntryFrequency();
+ return (isDef + isUse) * (freq.getFrequency() * Scale);
}
LiveRange LiveIntervals::addLiveRangeToEndOfBlock(unsigned reg,
diff --git a/lib/CodeGen/LiveRangeEdit.cpp b/lib/CodeGen/LiveRangeEdit.cpp
index 7793e96..b1de6a0 100644
--- a/lib/CodeGen/LiveRangeEdit.cpp
+++ b/lib/CodeGen/LiveRangeEdit.cpp
@@ -374,9 +374,11 @@
}
}
-void LiveRangeEdit::calculateRegClassAndHint(MachineFunction &MF,
- const MachineLoopInfo &Loops) {
- VirtRegAuxInfo VRAI(MF, LIS, Loops);
+void
+LiveRangeEdit::calculateRegClassAndHint(MachineFunction &MF,
+ const MachineLoopInfo &Loops,
+ const MachineBlockFrequencyInfo &MBFI) {
+ VirtRegAuxInfo VRAI(MF, LIS, Loops, MBFI);
for (iterator I = begin(), E = end(); I != E; ++I) {
LiveInterval &LI = **I;
if (MRI.recomputeRegClass(LI.reg, MF.getTarget()))
diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp
index 7fcfe9e..d6a7d6f 100644
--- a/lib/CodeGen/RegAllocBasic.cpp
+++ b/lib/CodeGen/RegAllocBasic.cpp
@@ -24,6 +24,7 @@
#include "llvm/CodeGen/LiveRangeEdit.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
#include "llvm/CodeGen/LiveStackAnalysis.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
@@ -145,6 +146,8 @@
AU.addRequired<CalculateSpillWeights>();
AU.addRequired<LiveStacks>();
AU.addPreserved<LiveStacks>();
+ AU.addRequired<MachineBlockFrequencyInfo>();
+ AU.addPreserved<MachineBlockFrequencyInfo>();
AU.addRequiredID(MachineDominatorsID);
AU.addPreservedID(MachineDominatorsID);
AU.addRequired<MachineLoopInfo>();
diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp
index 4974828..bdcef6f 100644
--- a/lib/CodeGen/RegAllocGreedy.cpp
+++ b/lib/CodeGen/RegAllocGreedy.cpp
@@ -29,6 +29,7 @@
#include "llvm/CodeGen/LiveRangeEdit.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
#include "llvm/CodeGen/LiveStackAnalysis.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
@@ -71,6 +72,7 @@
// analyses
SlotIndexes *Indexes;
+ MachineBlockFrequencyInfo *MBFI;
MachineDominatorTree *DomTree;
MachineLoopInfo *Loops;
EdgeBundles *Bundles;
@@ -320,6 +322,8 @@
void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
+ AU.addRequired<MachineBlockFrequencyInfo>();
+ AU.addPreserved<MachineBlockFrequencyInfo>();
AU.addRequired<AliasAnalysis>();
AU.addPreserved<AliasAnalysis>();
AU.addRequired<LiveIntervals>();
@@ -1770,6 +1774,7 @@
getAnalysis<LiveIntervals>(),
getAnalysis<LiveRegMatrix>());
Indexes = &getAnalysis<SlotIndexes>();
+ MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
DomTree = &getAnalysis<MachineDominatorTree>();
SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM));
Loops = &getAnalysis<MachineLoopInfo>();
@@ -1778,7 +1783,7 @@
DebugVars = &getAnalysis<LiveDebugVariables>();
SA.reset(new SplitAnalysis(*VRM, *LIS, *Loops));
- SE.reset(new SplitEditor(*SA, *LIS, *VRM, *DomTree));
+ SE.reset(new SplitEditor(*SA, *LIS, *VRM, *DomTree, *MBFI));
ExtraRegInfo.clear();
ExtraRegInfo.resize(MRI->getNumVirtRegs());
NextCascade = 1;
diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp
index 15a88e2..ecfe6ad 100644
--- a/lib/CodeGen/RegAllocPBQP.cpp
+++ b/lib/CodeGen/RegAllocPBQP.cpp
@@ -40,9 +40,9 @@
#include "llvm/CodeGen/LiveIntervalAnalysis.h"
#include "llvm/CodeGen/LiveRangeEdit.h"
#include "llvm/CodeGen/LiveStackAnalysis.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/PBQP/Graph.h"
#include "llvm/CodeGen/PBQP/HeuristicSolver.h"
@@ -96,7 +96,6 @@
initializeLiveIntervalsPass(*PassRegistry::getPassRegistry());
initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry());
initializeLiveStacksPass(*PassRegistry::getPassRegistry());
- initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry());
initializeVirtRegMapPass(*PassRegistry::getPassRegistry());
}
@@ -130,8 +129,8 @@
const TargetMachine *tm;
const TargetRegisterInfo *tri;
const TargetInstrInfo *tii;
- const MachineLoopInfo *loopInfo;
MachineRegisterInfo *mri;
+ const MachineBlockFrequencyInfo *mbfi;
OwningPtr<Spiller> spiller;
LiveIntervals *lis;
@@ -188,7 +187,7 @@
}
PBQPRAProblem *PBQPBuilder::build(MachineFunction *mf, const LiveIntervals *lis,
- const MachineLoopInfo *loopInfo,
+ const MachineBlockFrequencyInfo *mbfi,
const RegSet &vregs) {
LiveIntervals *LIS = const_cast<LiveIntervals*>(lis);
@@ -313,10 +312,10 @@
PBQPRAProblem *PBQPBuilderWithCoalescing::build(MachineFunction *mf,
const LiveIntervals *lis,
- const MachineLoopInfo *loopInfo,
+ const MachineBlockFrequencyInfo *mbfi,
const RegSet &vregs) {
- OwningPtr<PBQPRAProblem> p(PBQPBuilder::build(mf, lis, loopInfo, vregs));
+ OwningPtr<PBQPRAProblem> p(PBQPBuilder::build(mf, lis, mbfi, vregs));
PBQP::Graph &g = p->getGraph();
const TargetMachine &tm = mf->getTarget();
@@ -350,7 +349,7 @@
PBQP::PBQPNum cBenefit =
copyFactor * LiveIntervals::getSpillWeight(false, true,
- loopInfo->getLoopDepth(mbb));
+ mbfi->getBlockFreq(mbb));
if (cp.isPhys()) {
if (!mf->getRegInfo().isAllocatable(dst)) {
@@ -435,10 +434,10 @@
au.addRequired<CalculateSpillWeights>();
au.addRequired<LiveStacks>();
au.addPreserved<LiveStacks>();
+ au.addRequired<MachineBlockFrequencyInfo>();
+ au.addPreserved<MachineBlockFrequencyInfo>();
au.addRequired<MachineDominatorTree>();
au.addPreserved<MachineDominatorTree>();
- au.addRequired<MachineLoopInfo>();
- au.addPreserved<MachineLoopInfo>();
au.addRequired<VirtRegMap>();
au.addPreserved<VirtRegMap>();
MachineFunctionPass::getAnalysisUsage(au);
@@ -546,7 +545,7 @@
lis = &getAnalysis<LiveIntervals>();
lss = &getAnalysis<LiveStacks>();
- loopInfo = &getAnalysis<MachineLoopInfo>();
+ mbfi = &getAnalysis<MachineBlockFrequencyInfo>();
vrm = &getAnalysis<VirtRegMap>();
spiller.reset(createInlineSpiller(*this, MF, *vrm));
@@ -584,7 +583,7 @@
DEBUG(dbgs() << " PBQP Regalloc round " << round << ":\n");
OwningPtr<PBQPRAProblem> problem(
- builder->build(mf, lis, loopInfo, vregsToAlloc));
+ builder->build(mf, lis, mbfi, vregsToAlloc));
#ifndef NDEBUG
if (pbqpDumpGraphs) {
diff --git a/lib/CodeGen/SpillPlacement.cpp b/lib/CodeGen/SpillPlacement.cpp
index c5bbba3..840f05b 100644
--- a/lib/CodeGen/SpillPlacement.cpp
+++ b/lib/CodeGen/SpillPlacement.cpp
@@ -31,8 +31,8 @@
#include "SpillPlacement.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/CodeGen/EdgeBundles.h"
-#include "llvm/CodeGen/LiveIntervalAnalysis.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/Passes.h"
@@ -53,6 +53,7 @@
void SpillPlacement::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
+ AU.addRequired<MachineBlockFrequencyInfo>();
AU.addRequiredTransitive<EdgeBundles>();
AU.addRequiredTransitive<MachineLoopInfo>();
MachineFunctionPass::getAnalysisUsage(AU);
@@ -178,9 +179,10 @@
// Compute total ingoing and outgoing block frequencies for all bundles.
BlockFrequency.resize(mf.getNumBlockIDs());
+ MachineBlockFrequencyInfo &MBFI = getAnalysis<MachineBlockFrequencyInfo>();
+ float EntryFreq = BlockFrequency::getEntryFrequency();
for (MachineFunction::iterator I = mf.begin(), E = mf.end(); I != E; ++I) {
- float Freq = LiveIntervals::getSpillWeight(true, false,
- loops->getLoopDepth(I));
+ float Freq = MBFI.getBlockFreq(I).getFrequency() / EntryFreq;
unsigned Num = I->getNumber();
BlockFrequency[Num] = Freq;
nodes[bundles->getBundle(Num, 1)].Scale[0] += Freq;
diff --git a/lib/CodeGen/SplitKit.cpp b/lib/CodeGen/SplitKit.cpp
index 0a3818e..e717fac 100644
--- a/lib/CodeGen/SplitKit.cpp
+++ b/lib/CodeGen/SplitKit.cpp
@@ -325,12 +325,14 @@
SplitEditor::SplitEditor(SplitAnalysis &sa,
LiveIntervals &lis,
VirtRegMap &vrm,
- MachineDominatorTree &mdt)
+ MachineDominatorTree &mdt,
+ MachineBlockFrequencyInfo &mbfi)
: SA(sa), LIS(lis), VRM(vrm),
MRI(vrm.getMachineFunction().getRegInfo()),
MDT(mdt),
TII(*vrm.getMachineFunction().getTarget().getInstrInfo()),
TRI(*vrm.getMachineFunction().getTarget().getRegisterInfo()),
+ MBFI(mbfi),
Edit(0),
OpenIdx(0),
SpillMode(SM_Partition),
@@ -1119,7 +1121,7 @@
}
// Calculate spill weight and allocation hints for new intervals.
- Edit->calculateRegClassAndHint(VRM.getMachineFunction(), SA.Loops);
+ Edit->calculateRegClassAndHint(VRM.getMachineFunction(), SA.Loops, MBFI);
assert(!LRMap || LRMap->size() == Edit->size());
}
diff --git a/lib/CodeGen/SplitKit.h b/lib/CodeGen/SplitKit.h
index 4005a3d..f029c73 100644
--- a/lib/CodeGen/SplitKit.h
+++ b/lib/CodeGen/SplitKit.h
@@ -27,6 +27,7 @@
class LiveInterval;
class LiveIntervals;
class LiveRangeEdit;
+class MachineBlockFrequencyInfo;
class MachineInstr;
class MachineLoopInfo;
class MachineRegisterInfo;
@@ -215,6 +216,7 @@
MachineDominatorTree &MDT;
const TargetInstrInfo &TII;
const TargetRegisterInfo &TRI;
+ const MachineBlockFrequencyInfo &MBFI;
public:
@@ -349,7 +351,7 @@
/// Create a new SplitEditor for editing the LiveInterval analyzed by SA.
/// Newly created intervals will be appended to newIntervals.
SplitEditor(SplitAnalysis &SA, LiveIntervals&, VirtRegMap&,
- MachineDominatorTree&);
+ MachineDominatorTree&, MachineBlockFrequencyInfo &);
/// reset - Prepare for a new split.
void reset(LiveRangeEdit&, ComplementSpillMode = SM_Partition);
diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp
index f951561..3c35b83 100644
--- a/lib/CodeGen/StackSlotColoring.cpp
+++ b/lib/CodeGen/StackSlotColoring.cpp
@@ -19,9 +19,9 @@
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/LiveIntervalAnalysis.h"
#include "llvm/CodeGen/LiveStackAnalysis.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
@@ -48,7 +48,7 @@
LiveStacks* LS;
MachineFrameInfo *MFI;
const TargetInstrInfo *TII;
- const MachineLoopInfo *loopInfo;
+ const MachineBlockFrequencyInfo *MBFI;
// SSIntervals - Spill slot intervals.
std::vector<LiveInterval*> SSIntervals;
@@ -89,8 +89,8 @@
AU.addRequired<SlotIndexes>();
AU.addPreserved<SlotIndexes>();
AU.addRequired<LiveStacks>();
- AU.addRequired<MachineLoopInfo>();
- AU.addPreserved<MachineLoopInfo>();
+ AU.addRequired<MachineBlockFrequencyInfo>();
+ AU.addPreserved<MachineBlockFrequencyInfo>();
AU.addPreservedID(MachineDominatorsID);
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -139,7 +139,7 @@
for (MachineFunction::iterator MBBI = MF.begin(), E = MF.end();
MBBI != E; ++MBBI) {
MachineBasicBlock *MBB = &*MBBI;
- unsigned loopDepth = loopInfo->getLoopDepth(MBB);
+ BlockFrequency Freq = MBFI->getBlockFreq(MBB);
for (MachineBasicBlock::iterator MII = MBB->begin(), EE = MBB->end();
MII != EE; ++MII) {
MachineInstr *MI = &*MII;
@@ -154,7 +154,7 @@
continue;
LiveInterval &li = LS->getInterval(FI);
if (!MI->isDebugValue())
- li.weight += LiveIntervals::getSpillWeight(false, true, loopDepth);
+ li.weight += LiveIntervals::getSpillWeight(false, true, Freq);
SSRefs[FI].push_back(MI);
}
}
@@ -396,7 +396,7 @@
MFI = MF.getFrameInfo();
TII = MF.getTarget().getInstrInfo();
LS = &getAnalysis<LiveStacks>();
- loopInfo = &getAnalysis<MachineLoopInfo>();
+ MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
bool Changed = false;