Disable hoisting MI to hotter basic blocks
In current Hoist() function of machine licm pass, it will not check the source and destination basic block frequencies that a instruction is hoisted from/to.
There is a chance that instruction is hoisted from a cold to a hot basic block.
In this patch, we add options to disable machine instruction hoisting if destination block is hotter.
Differential Revision: https://reviews.llvm.org/D63676
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 6a898ff..194125f 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -23,6 +23,7 @@
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -74,6 +75,27 @@
HoistConstStores("hoist-const-stores",
cl::desc("Hoist invariant stores"),
cl::init(true), cl::Hidden);
+// The default threshold of 100 (i.e. if target block is 100 times hotter)
+// is based on empirical data on a single target and is subject to tuning.
+static cl::opt<unsigned>
+BlockFrequencyRatioThreshold("block-freq-ratio-threshold",
+ cl::desc("Do not hoist instructions if target"
+ "block is N times hotter than the source."),
+ cl::init(100), cl::Hidden);
+
+enum class UseBFI { None, PGO, All };
+
+static cl::opt<UseBFI>
+DisableHoistingToHotterBlocks("disable-hoisting-to-hotter-blocks",
+ cl::desc("Disable hoisting instructions to"
+ " hotter blocks"),
+ cl::init(UseBFI::None), cl::Hidden,
+ cl::values(clEnumValN(UseBFI::None, "none",
+ "disable the feature"),
+ clEnumValN(UseBFI::PGO, "pgo",
+ "enable the feature when using profile data"),
+ clEnumValN(UseBFI::All, "all",
+ "enable the feature with/wo profile data")));
STATISTIC(NumHoisted,
"Number of machine instructions hoisted out of loops");
@@ -87,6 +109,8 @@
"Number of machine instructions hoisted out of loops post regalloc");
STATISTIC(NumStoreConst,
"Number of stores of const phys reg hoisted out of loops");
+STATISTIC(NumNotHoistedDueToHotness,
+ "Number of instructions not hoisted due to block frequency");
namespace {
@@ -98,9 +122,11 @@
MachineRegisterInfo *MRI;
TargetSchedModel SchedModel;
bool PreRegAlloc;
+ bool HasProfileData;
// Various analyses that we use...
AliasAnalysis *AA; // Alias analysis info.
+ MachineBlockFrequencyInfo *MBFI; // Machine block frequncy info
MachineLoopInfo *MLI; // Current MachineLoopInfo
MachineDominatorTree *DT; // Machine dominator tree for the cur loop
@@ -150,6 +176,8 @@
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineLoopInfo>();
+ if (DisableHoistingToHotterBlocks != UseBFI::None)
+ AU.addRequired<MachineBlockFrequencyInfo>();
AU.addRequired<MachineDominatorTree>();
AU.addRequired<AAResultsWrapperPass>();
AU.addPreserved<MachineLoopInfo>();
@@ -245,6 +273,8 @@
void InitCSEMap(MachineBasicBlock *BB);
+ bool isTgtHotterThanSrc(MachineBasicBlock *SrcBlock,
+ MachineBasicBlock *TgtBlock);
MachineBasicBlock *getCurPreheader();
};
@@ -275,6 +305,7 @@
INITIALIZE_PASS_BEGIN(MachineLICM, DEBUG_TYPE,
"Machine Loop Invariant Code Motion", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
INITIALIZE_PASS_END(MachineLICM, DEBUG_TYPE,
@@ -283,6 +314,7 @@
INITIALIZE_PASS_BEGIN(EarlyMachineLICM, "early-machinelicm",
"Early Machine Loop Invariant Code Motion", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
INITIALIZE_PASS_END(EarlyMachineLICM, "early-machinelicm",
@@ -315,6 +347,7 @@
SchedModel.init(&ST);
PreRegAlloc = MRI->isSSA();
+ HasProfileData = MF.getFunction().hasProfileData();
if (PreRegAlloc)
LLVM_DEBUG(dbgs() << "******** Pre-regalloc Machine LICM: ");
@@ -333,6 +366,8 @@
}
// Get our Loop information...
+ if (DisableHoistingToHotterBlocks != UseBFI::None)
+ MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
MLI = &getAnalysis<MachineLoopInfo>();
DT = &getAnalysis<MachineDominatorTree>();
AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
@@ -1433,6 +1468,15 @@
/// that are safe to hoist, this instruction is called to do the dirty work.
/// It returns true if the instruction is hoisted.
bool MachineLICMBase::Hoist(MachineInstr *MI, MachineBasicBlock *Preheader) {
+ MachineBasicBlock *SrcBlock = MI->getParent();
+
+ // Disable the instruction hoisting due to block hotness
+ if ((DisableHoistingToHotterBlocks == UseBFI::All ||
+ (DisableHoistingToHotterBlocks == UseBFI::PGO && HasProfileData)) &&
+ isTgtHotterThanSrc(SrcBlock, Preheader)) {
+ ++NumNotHoistedDueToHotness;
+ return false;
+ }
// First check whether we should hoist this instruction.
if (!IsLoopInvariantInst(*MI) || !IsProfitableToHoist(*MI)) {
// If not, try unfolding a hoistable load.
@@ -1526,3 +1570,21 @@
}
return CurPreheader;
}
+
+/// Is the target basic block at least "BlockFrequencyRatioThreshold"
+/// times hotter than the source basic block.
+bool MachineLICMBase::isTgtHotterThanSrc(MachineBasicBlock *SrcBlock,
+ MachineBasicBlock *TgtBlock) {
+ // Parse source and target basic block frequency from MBFI
+ uint64_t SrcBF = MBFI->getBlockFreq(SrcBlock).getFrequency();
+ uint64_t DstBF = MBFI->getBlockFreq(TgtBlock).getFrequency();
+
+ // Disable the hoisting if source block frequency is zero
+ if (!SrcBF)
+ return true;
+
+ double Ratio = (double)DstBF / SrcBF;
+
+ // Compare the block frequency ratio with the threshold
+ return Ratio > BlockFrequencyRatioThreshold;
+}