blob: 93d2b7550a01441cf97bb77c9ad0c85e9a66fdcb [file] [log] [blame]
Aditya Kumar801394a2018-09-07 15:03:49 +00001//===- HotColdSplitting.cpp -- Outline Cold Regions -------------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Outline cold regions to a separate function.
11// TODO: Update BFI and BPI
12// TODO: Add all the outlined functions to a separate section.
13//
14//===----------------------------------------------------------------------===//
15
Vedant Kumar03aaa3e2018-12-07 20:23:52 +000016#include "llvm/ADT/PostOrderIterator.h"
Aditya Kumar801394a2018-09-07 15:03:49 +000017#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/Analysis/AliasAnalysis.h"
20#include "llvm/Analysis/BlockFrequencyInfo.h"
21#include "llvm/Analysis/BranchProbabilityInfo.h"
22#include "llvm/Analysis/CFG.h"
23#include "llvm/Analysis/OptimizationRemarkEmitter.h"
24#include "llvm/Analysis/PostDominators.h"
25#include "llvm/Analysis/ProfileSummaryInfo.h"
Sebastian Popa1f20fc2018-09-10 15:08:02 +000026#include "llvm/Analysis/TargetTransformInfo.h"
Aditya Kumar801394a2018-09-07 15:03:49 +000027#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/CFG.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/Dominators.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/Instructions.h"
Vedant Kumardd4be532018-10-29 19:15:39 +000035#include "llvm/IR/IntrinsicInst.h"
Aditya Kumar801394a2018-09-07 15:03:49 +000036#include "llvm/IR/Metadata.h"
37#include "llvm/IR/Module.h"
38#include "llvm/IR/PassManager.h"
39#include "llvm/IR/Type.h"
40#include "llvm/IR/Use.h"
41#include "llvm/IR/User.h"
42#include "llvm/IR/Value.h"
43#include "llvm/Pass.h"
44#include "llvm/Support/BlockFrequency.h"
45#include "llvm/Support/BranchProbability.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/raw_ostream.h"
48#include "llvm/Transforms/IPO.h"
Aditya Kumar9e20ade2018-10-03 05:55:20 +000049#include "llvm/Transforms/IPO/HotColdSplitting.h"
Aditya Kumar801394a2018-09-07 15:03:49 +000050#include "llvm/Transforms/Scalar.h"
51#include "llvm/Transforms/Utils/BasicBlockUtils.h"
52#include "llvm/Transforms/Utils/Cloning.h"
53#include "llvm/Transforms/Utils/CodeExtractor.h"
54#include "llvm/Transforms/Utils/Local.h"
55#include "llvm/Transforms/Utils/SSAUpdater.h"
56#include "llvm/Transforms/Utils/ValueMapper.h"
57#include <algorithm>
58#include <cassert>
59
60#define DEBUG_TYPE "hotcoldsplit"
61
Vedant Kumarc2990062018-10-24 22:15:41 +000062STATISTIC(NumColdRegionsFound, "Number of cold regions found.");
63STATISTIC(NumColdRegionsOutlined, "Number of cold regions outlined.");
Aditya Kumar801394a2018-09-07 15:03:49 +000064
65using namespace llvm;
66
67static cl::opt<bool> EnableStaticAnalyis("hot-cold-static-analysis",
68 cl::init(true), cl::Hidden);
69
Vedant Kumard2a895a2018-11-04 23:11:57 +000070static cl::opt<int>
71 MinOutliningThreshold("min-outlining-thresh", cl::init(3), cl::Hidden,
72 cl::desc("Code size threshold for outlining within a "
73 "single BB (as a multiple of TCC_Basic)"));
Aditya Kumar801394a2018-09-07 15:03:49 +000074
75namespace {
76
77struct PostDomTree : PostDomTreeBase<BasicBlock> {
78 PostDomTree(Function &F) { recalculate(F); }
79};
80
Vedant Kumarc2990062018-10-24 22:15:41 +000081/// A sequence of basic blocks.
82///
83/// A 0-sized SmallVector is slightly cheaper to move than a std::vector.
84using BlockSequence = SmallVector<BasicBlock *, 0>;
Aditya Kumar801394a2018-09-07 15:03:49 +000085
Sebastian Pop542e5222018-10-15 21:43:11 +000086// Same as blockEndsInUnreachable in CodeGen/BranchFolding.cpp. Do not modify
87// this function unless you modify the MBB version as well.
88//
89/// A no successor, non-return block probably ends in unreachable and is cold.
90/// Also consider a block that ends in an indirect branch to be a return block,
91/// since many targets use plain indirect branches to return.
Aditya Kumar801394a2018-09-07 15:03:49 +000092bool blockEndsInUnreachable(const BasicBlock &BB) {
Sebastian Pop542e5222018-10-15 21:43:11 +000093 if (!succ_empty(&BB))
94 return false;
Aditya Kumar801394a2018-09-07 15:03:49 +000095 if (BB.empty())
96 return true;
Chandler Carruthedb12a82018-10-15 10:04:59 +000097 const Instruction *I = BB.getTerminator();
Sebastian Pop542e5222018-10-15 21:43:11 +000098 return !(isa<ReturnInst>(I) || isa<IndirectBrInst>(I));
Aditya Kumar801394a2018-09-07 15:03:49 +000099}
100
Aditya Kumara27014b2018-10-03 06:21:05 +0000101static bool exceptionHandlingFunctions(const CallInst *CI) {
102 auto F = CI->getCalledFunction();
103 if (!F)
104 return false;
105 auto FName = F->getName();
106 return FName == "__cxa_begin_catch" ||
107 FName == "__cxa_free_exception" ||
108 FName == "__cxa_allocate_exception" ||
109 FName == "__cxa_begin_catch" ||
110 FName == "__cxa_end_catch";
111}
112
Sebastian Pop542e5222018-10-15 21:43:11 +0000113static bool unlikelyExecuted(const BasicBlock &BB) {
Aditya Kumar801394a2018-09-07 15:03:49 +0000114 if (blockEndsInUnreachable(BB))
115 return true;
116 // Exception handling blocks are unlikely executed.
117 if (BB.isEHPad())
118 return true;
119 for (const Instruction &I : BB)
120 if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
121 // The block is cold if it calls functions tagged as cold or noreturn.
122 if (CI->hasFnAttr(Attribute::Cold) ||
Aditya Kumara27014b2018-10-03 06:21:05 +0000123 CI->hasFnAttr(Attribute::NoReturn) ||
124 exceptionHandlingFunctions(CI))
Aditya Kumar801394a2018-09-07 15:03:49 +0000125 return true;
126
127 // Assume that inline assembly is hot code.
128 if (isa<InlineAsm>(CI->getCalledValue()))
129 return false;
130 }
131 return false;
132}
133
Vedant Kumarc2990062018-10-24 22:15:41 +0000134/// Check whether it's safe to outline \p BB.
135static bool mayExtractBlock(const BasicBlock &BB) {
136 return !BB.hasAddressTaken();
Sebastian Pop542e5222018-10-15 21:43:11 +0000137}
138
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000139/// Check whether \p Region is profitable to outline.
140static bool isProfitableToOutline(const BlockSequence &Region,
Vedant Kumard2a895a2018-11-04 23:11:57 +0000141 TargetTransformInfo &TTI) {
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000142 if (Region.size() > 1)
143 return true;
144
Vedant Kumard2a895a2018-11-04 23:11:57 +0000145 int Cost = 0;
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000146 const BasicBlock &BB = *Region[0];
Vedant Kumardd4be532018-10-29 19:15:39 +0000147 for (const Instruction &I : BB) {
148 if (isa<DbgInfoIntrinsic>(&I) || &I == BB.getTerminator())
149 continue;
Vedant Kumard2a895a2018-11-04 23:11:57 +0000150
151 Cost += TTI.getInstructionCost(&I, TargetTransformInfo::TCK_CodeSize);
152
153 if (Cost >= (MinOutliningThreshold * TargetTransformInfo::TCC_Basic))
Vedant Kumardd4be532018-10-29 19:15:39 +0000154 return true;
155 }
156 return false;
157}
158
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000159/// Mark \p F cold. Return true if it's changed.
160static bool markEntireFunctionCold(Function &F) {
161 assert(!F.hasFnAttribute(Attribute::OptimizeNone) && "Can't mark this cold");
162 bool Changed = false;
163 if (!F.hasFnAttribute(Attribute::MinSize)) {
164 F.addFnAttr(Attribute::MinSize);
165 Changed = true;
Aditya Kumar801394a2018-09-07 15:03:49 +0000166 }
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000167 // TODO: Move this function into a cold section.
168 return Changed;
Aditya Kumar801394a2018-09-07 15:03:49 +0000169}
170
171class HotColdSplitting {
172public:
173 HotColdSplitting(ProfileSummaryInfo *ProfSI,
174 function_ref<BlockFrequencyInfo *(Function &)> GBFI,
Sebastian Popa1f20fc2018-09-10 15:08:02 +0000175 function_ref<TargetTransformInfo &(Function &)> GTTI,
Aditya Kumar801394a2018-09-07 15:03:49 +0000176 std::function<OptimizationRemarkEmitter &(Function &)> *GORE)
Sebastian Popa1f20fc2018-09-10 15:08:02 +0000177 : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE) {}
Aditya Kumar801394a2018-09-07 15:03:49 +0000178 bool run(Module &M);
179
180private:
181 bool shouldOutlineFrom(const Function &F) const;
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000182 bool outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
183 BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
184 DominatorTree &DT, PostDomTree &PDT,
185 OptimizationRemarkEmitter &ORE);
Vedant Kumarc2990062018-10-24 22:15:41 +0000186 Function *extractColdRegion(const BlockSequence &Region, DominatorTree &DT,
Vedant Kumard2a895a2018-11-04 23:11:57 +0000187 BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
Teresa Johnsonc8dba682018-10-24 18:53:47 +0000188 OptimizationRemarkEmitter &ORE, unsigned Count);
Aditya Kumar801394a2018-09-07 15:03:49 +0000189 SmallPtrSet<const Function *, 2> OutlinedFunctions;
190 ProfileSummaryInfo *PSI;
191 function_ref<BlockFrequencyInfo *(Function &)> GetBFI;
Sebastian Popa1f20fc2018-09-10 15:08:02 +0000192 function_ref<TargetTransformInfo &(Function &)> GetTTI;
Aditya Kumar801394a2018-09-07 15:03:49 +0000193 std::function<OptimizationRemarkEmitter &(Function &)> *GetORE;
194};
195
196class HotColdSplittingLegacyPass : public ModulePass {
197public:
198 static char ID;
199 HotColdSplittingLegacyPass() : ModulePass(ID) {
200 initializeHotColdSplittingLegacyPassPass(*PassRegistry::getPassRegistry());
201 }
202
203 void getAnalysisUsage(AnalysisUsage &AU) const override {
204 AU.addRequired<AssumptionCacheTracker>();
205 AU.addRequired<BlockFrequencyInfoWrapperPass>();
206 AU.addRequired<ProfileSummaryInfoWrapperPass>();
Sebastian Popa1f20fc2018-09-10 15:08:02 +0000207 AU.addRequired<TargetTransformInfoWrapperPass>();
Aditya Kumar801394a2018-09-07 15:03:49 +0000208 }
209
210 bool runOnModule(Module &M) override;
211};
212
213} // end anonymous namespace
214
215// Returns false if the function should not be considered for hot-cold split
Sebastian Pop0f30f082018-09-14 20:36:19 +0000216// optimization.
Aditya Kumar801394a2018-09-07 15:03:49 +0000217bool HotColdSplitting::shouldOutlineFrom(const Function &F) const {
Sebastian Pop0f30f082018-09-14 20:36:19 +0000218 // Do not try to outline again from an already outlined cold function.
219 if (OutlinedFunctions.count(&F))
220 return false;
221
Aditya Kumar801394a2018-09-07 15:03:49 +0000222 if (F.size() <= 2)
223 return false;
224
Vedant Kumarc2990062018-10-24 22:15:41 +0000225 // TODO: Consider only skipping functions marked `optnone` or `cold`.
226
Aditya Kumar801394a2018-09-07 15:03:49 +0000227 if (F.hasAddressTaken())
228 return false;
229
230 if (F.hasFnAttribute(Attribute::AlwaysInline))
231 return false;
232
233 if (F.hasFnAttribute(Attribute::NoInline))
234 return false;
235
236 if (F.getCallingConv() == CallingConv::Cold)
237 return false;
238
239 if (PSI->isFunctionEntryCold(&F))
240 return false;
241 return true;
242}
243
Vedant Kumarc2990062018-10-24 22:15:41 +0000244Function *HotColdSplitting::extractColdRegion(const BlockSequence &Region,
245 DominatorTree &DT,
246 BlockFrequencyInfo *BFI,
Vedant Kumard2a895a2018-11-04 23:11:57 +0000247 TargetTransformInfo &TTI,
Vedant Kumarc2990062018-10-24 22:15:41 +0000248 OptimizationRemarkEmitter &ORE,
249 unsigned Count) {
Teresa Johnsonf431a2f2018-10-22 19:06:42 +0000250 assert(!Region.empty());
Sebastian Pop12171602018-09-14 20:36:10 +0000251
Aditya Kumar801394a2018-09-07 15:03:49 +0000252 // TODO: Pass BFI and BPI to update profile information.
Vedant Kumarc2990062018-10-24 22:15:41 +0000253 CodeExtractor CE(Region, &DT, /* AggregateArgs */ false, /* BFI */ nullptr,
Teresa Johnsonc8dba682018-10-24 18:53:47 +0000254 /* BPI */ nullptr, /* AllowVarArgs */ false,
255 /* AllowAlloca */ false,
256 /* Suffix */ "cold." + std::to_string(Count));
Aditya Kumar801394a2018-09-07 15:03:49 +0000257
258 SetVector<Value *> Inputs, Outputs, Sinks;
259 CE.findInputsOutputs(Inputs, Outputs, Sinks);
260
261 // Do not extract regions that have live exit variables.
Vedant Kumarc2990062018-10-24 22:15:41 +0000262 if (Outputs.size() > 0) {
263 LLVM_DEBUG(llvm::dbgs() << "Not outlining; live outputs\n");
Aditya Kumar801394a2018-09-07 15:03:49 +0000264 return nullptr;
Vedant Kumarc2990062018-10-24 22:15:41 +0000265 }
Aditya Kumar801394a2018-09-07 15:03:49 +0000266
Vedant Kumarc2990062018-10-24 22:15:41 +0000267 // TODO: Run MergeBasicBlockIntoOnlyPred on the outlined function.
Teresa Johnsonf431a2f2018-10-22 19:06:42 +0000268 Function *OrigF = Region[0]->getParent();
Aditya Kumar801394a2018-09-07 15:03:49 +0000269 if (Function *OutF = CE.extractCodeRegion()) {
270 User *U = *OutF->user_begin();
271 CallInst *CI = cast<CallInst>(U);
272 CallSite CS(CI);
Vedant Kumarc2990062018-10-24 22:15:41 +0000273 NumColdRegionsOutlined++;
Vedant Kumard2a895a2018-11-04 23:11:57 +0000274 if (TTI.useColdCCForColdCall(*OutF)) {
Sebastian Popa1f20fc2018-09-10 15:08:02 +0000275 OutF->setCallingConv(CallingConv::Cold);
276 CS.setCallingConv(CallingConv::Cold);
277 }
Aditya Kumar801394a2018-09-07 15:03:49 +0000278 CI->setIsNoInline();
Vedant Kumar50315462018-10-23 19:41:12 +0000279
280 // Try to make the outlined code as small as possible on the assumption
281 // that it's cold.
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000282 markEntireFunctionCold(*OutF);
Vedant Kumar50315462018-10-23 19:41:12 +0000283
Sebastian Pop0f30f082018-09-14 20:36:19 +0000284 LLVM_DEBUG(llvm::dbgs() << "Outlined Region: " << *OutF);
Teresa Johnsonf431a2f2018-10-22 19:06:42 +0000285 ORE.emit([&]() {
286 return OptimizationRemark(DEBUG_TYPE, "HotColdSplit",
287 &*Region[0]->begin())
288 << ore::NV("Original", OrigF) << " split cold code into "
289 << ore::NV("Split", OutF);
290 });
Aditya Kumar801394a2018-09-07 15:03:49 +0000291 return OutF;
292 }
293
294 ORE.emit([&]() {
295 return OptimizationRemarkMissed(DEBUG_TYPE, "ExtractFailed",
296 &*Region[0]->begin())
297 << "Failed to extract region at block "
298 << ore::NV("Block", Region.front());
299 });
300 return nullptr;
301}
302
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000303/// A pair of (basic block, score).
304using BlockTy = std::pair<BasicBlock *, unsigned>;
305
306/// A maximal outlining region. This contains all blocks post-dominated by a
307/// sink block, the sink block itself, and all blocks dominated by the sink.
308class OutliningRegion {
309 /// A list of (block, score) pairs. A block's score is non-zero iff it's a
310 /// viable sub-region entry point. Blocks with higher scores are better entry
311 /// points (i.e. they are more distant ancestors of the sink block).
312 SmallVector<BlockTy, 0> Blocks = {};
313
314 /// The suggested entry point into the region. If the region has multiple
315 /// entry points, all blocks within the region may not be reachable from this
316 /// entry point.
317 BasicBlock *SuggestedEntryPoint = nullptr;
318
319 /// Whether the entire function is cold.
320 bool EntireFunctionCold = false;
321
322 /// Whether or not \p BB could be the entry point of an extracted region.
323 static bool isViableEntryPoint(BasicBlock &BB) { return !BB.isEHPad(); }
324
325 /// If \p BB is a viable entry point, return \p Score. Return 0 otherwise.
326 static unsigned getEntryPointScore(BasicBlock &BB, unsigned Score) {
327 return isViableEntryPoint(BB) ? Score : 0;
328 }
329
330 /// These scores should be lower than the score for predecessor blocks,
331 /// because regions starting at predecessor blocks are typically larger.
332 static constexpr unsigned ScoreForSuccBlock = 1;
333 static constexpr unsigned ScoreForSinkBlock = 1;
334
335 OutliningRegion(const OutliningRegion &) = delete;
336 OutliningRegion &operator=(const OutliningRegion &) = delete;
337
338public:
339 OutliningRegion() = default;
340 OutliningRegion(OutliningRegion &&) = default;
341 OutliningRegion &operator=(OutliningRegion &&) = default;
342
343 static OutliningRegion create(BasicBlock &SinkBB, const DominatorTree &DT,
344 const PostDomTree &PDT) {
345 OutliningRegion ColdRegion;
346
347 SmallPtrSet<BasicBlock *, 4> RegionBlocks;
348
349 auto addBlockToRegion = [&](BasicBlock *BB, unsigned Score) {
350 RegionBlocks.insert(BB);
351 ColdRegion.Blocks.emplace_back(BB, Score);
352 assert(RegionBlocks.size() == ColdRegion.Blocks.size() && "Duplicate BB");
353 };
354
355 // The ancestor farthest-away from SinkBB, and also post-dominated by it.
356 unsigned SinkScore = getEntryPointScore(SinkBB, ScoreForSinkBlock);
357 ColdRegion.SuggestedEntryPoint = (SinkScore > 0) ? &SinkBB : nullptr;
358 unsigned BestScore = SinkScore;
359
360 // Visit SinkBB's ancestors using inverse DFS.
361 auto PredIt = ++idf_begin(&SinkBB);
362 auto PredEnd = idf_end(&SinkBB);
363 while (PredIt != PredEnd) {
364 BasicBlock &PredBB = **PredIt;
365 bool SinkPostDom = PDT.dominates(&SinkBB, &PredBB);
366
367 // If the predecessor is cold and has no predecessors, the entire
368 // function must be cold.
369 if (SinkPostDom && pred_empty(&PredBB)) {
370 ColdRegion.EntireFunctionCold = true;
371 return ColdRegion;
372 }
373
374 // If SinkBB does not post-dominate a predecessor, do not mark the
375 // predecessor (or any of its predecessors) cold.
376 if (!SinkPostDom || !mayExtractBlock(PredBB)) {
377 PredIt.skipChildren();
378 continue;
379 }
380
381 // Keep track of the post-dominated ancestor farthest away from the sink.
382 // The path length is always >= 2, ensuring that predecessor blocks are
383 // considered as entry points before the sink block.
384 unsigned PredScore = getEntryPointScore(PredBB, PredIt.getPathLength());
385 if (PredScore > BestScore) {
386 ColdRegion.SuggestedEntryPoint = &PredBB;
387 BestScore = PredScore;
388 }
389
390 addBlockToRegion(&PredBB, PredScore);
391 ++PredIt;
Vedant Kumarc2990062018-10-24 22:15:41 +0000392 }
393
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000394 // Add SinkBB to the cold region. It's considered as an entry point before
395 // any sink-successor blocks.
396 addBlockToRegion(&SinkBB, SinkScore);
397
398 // Find all successors of SinkBB dominated by SinkBB using DFS.
399 auto SuccIt = ++df_begin(&SinkBB);
400 auto SuccEnd = df_end(&SinkBB);
401 while (SuccIt != SuccEnd) {
402 BasicBlock &SuccBB = **SuccIt;
403 bool SinkDom = DT.dominates(&SinkBB, &SuccBB);
404
405 // Don't allow the backwards & forwards DFSes to mark the same block.
406 bool DuplicateBlock = RegionBlocks.count(&SuccBB);
407
408 // If SinkBB does not dominate a successor, do not mark the successor (or
409 // any of its successors) cold.
410 if (DuplicateBlock || !SinkDom || !mayExtractBlock(SuccBB)) {
411 SuccIt.skipChildren();
412 continue;
413 }
414
415 unsigned SuccScore = getEntryPointScore(SuccBB, ScoreForSuccBlock);
416 if (SuccScore > BestScore) {
417 ColdRegion.SuggestedEntryPoint = &SuccBB;
418 BestScore = SuccScore;
419 }
420
421 addBlockToRegion(&SuccBB, SuccScore);
422 ++SuccIt;
423 }
424
425 return ColdRegion;
426 }
427
428 /// Whether this region has nothing to extract.
429 bool empty() const { return !SuggestedEntryPoint; }
430
431 /// The blocks in this region.
432 ArrayRef<std::pair<BasicBlock *, unsigned>> blocks() const { return Blocks; }
433
434 /// Whether the entire function containing this region is cold.
435 bool isEntireFunctionCold() const { return EntireFunctionCold; }
436
437 /// Remove a sub-region from this region and return it as a block sequence.
438 BlockSequence takeSingleEntrySubRegion(DominatorTree &DT) {
439 assert(!empty() && !isEntireFunctionCold() && "Nothing to extract");
440
441 // Remove blocks dominated by the suggested entry point from this region.
442 // During the removal, identify the next best entry point into the region.
443 // Ensure that the first extracted block is the suggested entry point.
444 BlockSequence SubRegion = {SuggestedEntryPoint};
445 BasicBlock *NextEntryPoint = nullptr;
446 unsigned NextScore = 0;
447 auto RegionEndIt = Blocks.end();
448 auto RegionStartIt = remove_if(Blocks, [&](const BlockTy &Block) {
449 BasicBlock *BB = Block.first;
450 unsigned Score = Block.second;
451 bool InSubRegion =
452 BB == SuggestedEntryPoint || DT.dominates(SuggestedEntryPoint, BB);
453 if (!InSubRegion && Score > NextScore) {
454 NextEntryPoint = BB;
455 NextScore = Score;
456 }
457 if (InSubRegion && BB != SuggestedEntryPoint)
458 SubRegion.push_back(BB);
459 return InSubRegion;
460 });
461 Blocks.erase(RegionStartIt, RegionEndIt);
462
463 // Update the suggested entry point.
464 SuggestedEntryPoint = NextEntryPoint;
465
466 return SubRegion;
467 }
468};
469
470bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
471 BlockFrequencyInfo *BFI,
472 TargetTransformInfo &TTI,
473 DominatorTree &DT, PostDomTree &PDT,
474 OptimizationRemarkEmitter &ORE) {
475 bool Changed = false;
476
477 // The set of cold blocks.
478 SmallPtrSet<BasicBlock *, 4> ColdBlocks;
479
480 // The worklist of non-intersecting regions left to outline.
481 SmallVector<OutliningRegion, 2> OutliningWorklist;
482
483 // Set up an RPO traversal. Experimentally, this performs better (outlines
484 // more) than a PO traversal, because we prevent region overlap by keeping
485 // the first region to contain a block.
486 ReversePostOrderTraversal<Function *> RPOT(&F);
487
488 // Find all cold regions.
489 for (BasicBlock *BB : RPOT) {
490 // Skip blocks which can't be outlined.
491 if (!mayExtractBlock(*BB))
492 continue;
493
494 // This block is already part of some outlining region.
495 if (ColdBlocks.count(BB))
496 continue;
497
498 bool Cold = PSI.isColdBlock(BB, BFI) ||
499 (EnableStaticAnalyis && unlikelyExecuted(*BB));
500 if (!Cold)
501 continue;
502
503 LLVM_DEBUG({
504 dbgs() << "Found a cold block:\n";
505 BB->dump();
506 });
507
508 auto Region = OutliningRegion::create(*BB, DT, PDT);
509 if (Region.empty())
510 continue;
511
512 if (Region.isEntireFunctionCold()) {
513 LLVM_DEBUG(dbgs() << "Entire function is cold\n");
514 return markEntireFunctionCold(F);
515 }
516
517 // If this outlining region intersects with another, drop the new region.
518 //
519 // TODO: It's theoretically possible to outline more by only keeping the
520 // largest region which contains a block, but the extra bookkeeping to do
521 // this is tricky/expensive.
522 bool RegionsOverlap = any_of(Region.blocks(), [&](const BlockTy &Block) {
523 return !ColdBlocks.insert(Block.first).second;
524 });
525 if (RegionsOverlap)
526 continue;
527
528 OutliningWorklist.emplace_back(std::move(Region));
529 ++NumColdRegionsFound;
530 }
531
532 // Outline single-entry cold regions, splitting up larger regions as needed.
533 unsigned OutlinedFunctionID = 1;
534 while (!OutliningWorklist.empty()) {
535 OutliningRegion Region = OutliningWorklist.pop_back_val();
536 assert(!Region.empty() && "Empty outlining region in worklist");
537 do {
538 BlockSequence SubRegion = Region.takeSingleEntrySubRegion(DT);
539 if (!isProfitableToOutline(SubRegion, TTI)) {
540 LLVM_DEBUG({
541 dbgs() << "Skipping outlining; not profitable to outline\n";
542 SubRegion[0]->dump();
543 });
544 continue;
545 }
546
547 LLVM_DEBUG({
548 dbgs() << "Hot/cold splitting attempting to outline these blocks:\n";
549 for (BasicBlock *BB : SubRegion)
550 BB->dump();
551 });
552
553 Function *Outlined =
554 extractColdRegion(SubRegion, DT, BFI, TTI, ORE, OutlinedFunctionID);
555 if (Outlined) {
556 ++OutlinedFunctionID;
557 OutlinedFunctions.insert(Outlined);
558 Changed = true;
559 }
560 } while (!Region.empty());
561 }
562
563 return Changed;
564}
565
566bool HotColdSplitting::run(Module &M) {
567 bool Changed = false;
568 OutlinedFunctions.clear();
569 for (auto &F : M) {
570 if (!shouldOutlineFrom(F)) {
571 LLVM_DEBUG(llvm::dbgs() << "Skipping " << F.getName() << "\n");
572 continue;
573 }
Vedant Kumarc2990062018-10-24 22:15:41 +0000574 LLVM_DEBUG(llvm::dbgs() << "Outlining in " << F.getName() << "\n");
Aditya Kumar801394a2018-09-07 15:03:49 +0000575 DominatorTree DT(F);
576 PostDomTree PDT(F);
577 PDT.recalculate(F);
Vedant Kumarc2990062018-10-24 22:15:41 +0000578 BlockFrequencyInfo *BFI = GetBFI(F);
Vedant Kumard2a895a2018-11-04 23:11:57 +0000579 TargetTransformInfo &TTI = GetTTI(F);
Vedant Kumarc2990062018-10-24 22:15:41 +0000580 OptimizationRemarkEmitter &ORE = (*GetORE)(F);
Vedant Kumar03aaa3e2018-12-07 20:23:52 +0000581 Changed |= outlineColdRegions(F, *PSI, BFI, TTI, DT, PDT, ORE);
Aditya Kumar801394a2018-09-07 15:03:49 +0000582 }
Vedant Kumarc2990062018-10-24 22:15:41 +0000583 return Changed;
Aditya Kumar801394a2018-09-07 15:03:49 +0000584}
585
586bool HotColdSplittingLegacyPass::runOnModule(Module &M) {
587 if (skipModule(M))
588 return false;
589 ProfileSummaryInfo *PSI =
Vedant Kumare7b789b2018-11-19 05:23:16 +0000590 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
Sebastian Popa1f20fc2018-09-10 15:08:02 +0000591 auto GTTI = [this](Function &F) -> TargetTransformInfo & {
592 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
593 };
Aditya Kumar801394a2018-09-07 15:03:49 +0000594 auto GBFI = [this](Function &F) {
595 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
596 };
597 std::unique_ptr<OptimizationRemarkEmitter> ORE;
598 std::function<OptimizationRemarkEmitter &(Function &)> GetORE =
599 [&ORE](Function &F) -> OptimizationRemarkEmitter & {
600 ORE.reset(new OptimizationRemarkEmitter(&F));
601 return *ORE.get();
602 };
603
Sebastian Popa1f20fc2018-09-10 15:08:02 +0000604 return HotColdSplitting(PSI, GBFI, GTTI, &GetORE).run(M);
Aditya Kumar801394a2018-09-07 15:03:49 +0000605}
606
Aditya Kumar9e20ade2018-10-03 05:55:20 +0000607PreservedAnalyses
608HotColdSplittingPass::run(Module &M, ModuleAnalysisManager &AM) {
609 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
610
611 std::function<AssumptionCache &(Function &)> GetAssumptionCache =
612 [&FAM](Function &F) -> AssumptionCache & {
613 return FAM.getResult<AssumptionAnalysis>(F);
614 };
615
616 auto GBFI = [&FAM](Function &F) {
617 return &FAM.getResult<BlockFrequencyAnalysis>(F);
618 };
619
620 std::function<TargetTransformInfo &(Function &)> GTTI =
621 [&FAM](Function &F) -> TargetTransformInfo & {
622 return FAM.getResult<TargetIRAnalysis>(F);
623 };
624
625 std::unique_ptr<OptimizationRemarkEmitter> ORE;
626 std::function<OptimizationRemarkEmitter &(Function &)> GetORE =
627 [&ORE](Function &F) -> OptimizationRemarkEmitter & {
628 ORE.reset(new OptimizationRemarkEmitter(&F));
629 return *ORE.get();
630 };
631
632 ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
633
634 if (HotColdSplitting(PSI, GBFI, GTTI, &GetORE).run(M))
635 return PreservedAnalyses::none();
636 return PreservedAnalyses::all();
637}
638
Aditya Kumar801394a2018-09-07 15:03:49 +0000639char HotColdSplittingLegacyPass::ID = 0;
640INITIALIZE_PASS_BEGIN(HotColdSplittingLegacyPass, "hotcoldsplit",
641 "Hot Cold Splitting", false, false)
642INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
643INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
644INITIALIZE_PASS_END(HotColdSplittingLegacyPass, "hotcoldsplit",
645 "Hot Cold Splitting", false, false)
646
647ModulePass *llvm::createHotColdSplittingPass() {
648 return new HotColdSplittingLegacyPass();
649}