blob: 4517b9b8f06ddb4d062ca64ffc922601770d8c45 [file] [log] [blame]
Fiona Glaserb417d462016-01-29 22:35:36 +00001//===--------- LoopSimplifyCFG.cpp - Loop CFG Simplification Pass ---------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the Loop SimplifyCFG Pass. This pass is responsible for
11// basic loop CFG cleanup, primarily to assist other loop passes. If you
12// encounter a noncanonical CFG construct that causes another loop pass to
13// perform suboptimally, this is the place to fix it up.
14//
15//===----------------------------------------------------------------------===//
16
Justin Bognerab6a5132016-05-03 21:47:32 +000017#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
Fiona Glaserb417d462016-01-29 22:35:36 +000018#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/Analysis/AliasAnalysis.h"
Daniel Jasperaec2fa32016-12-19 08:22:17 +000021#include "llvm/Analysis/AssumptionCache.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +000022#include "llvm/Analysis/BasicAliasAnalysis.h"
Fiona Glaserb417d462016-01-29 22:35:36 +000023#include "llvm/Analysis/DependenceAnalysis.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/LoopInfo.h"
26#include "llvm/Analysis/LoopPass.h"
Alina Sbirlea8b83d682018-08-22 20:10:21 +000027#include "llvm/Analysis/MemorySSA.h"
28#include "llvm/Analysis/MemorySSAUpdater.h"
Fiona Glaserb417d462016-01-29 22:35:36 +000029#include "llvm/Analysis/ScalarEvolution.h"
30#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
31#include "llvm/Analysis/TargetTransformInfo.h"
Chijun Sima21a8b602018-08-03 05:08:17 +000032#include "llvm/IR/DomTreeUpdater.h"
Fiona Glaserb417d462016-01-29 22:35:36 +000033#include "llvm/IR/Dominators.h"
Justin Bognerab6a5132016-05-03 21:47:32 +000034#include "llvm/Transforms/Scalar.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +000035#include "llvm/Transforms/Scalar/LoopPassManager.h"
David Blaikiea373d182018-03-28 17:44:36 +000036#include "llvm/Transforms/Utils.h"
Alina Sbirleadfd14ad2018-06-20 22:01:04 +000037#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/Local.h"
Chandler Carruth31088a92016-02-19 10:45:18 +000039#include "llvm/Transforms/Utils/LoopUtils.h"
Fiona Glaserb417d462016-01-29 22:35:36 +000040using namespace llvm;
41
42#define DEBUG_TYPE "loop-simplifycfg"
43
Max Kazantseve1c2dc22018-11-23 09:14:53 +000044static cl::opt<bool> EnableTermFolding("enable-loop-simplifycfg-term-folding",
Max Kazantsev530ff8f2018-12-28 06:22:39 +000045 cl::init(false));
Max Kazantseve1c2dc22018-11-23 09:14:53 +000046
Max Kazantsevc04b5302018-11-20 05:43:32 +000047STATISTIC(NumTerminatorsFolded,
48 "Number of terminators folded to unconditional branches");
Max Kazantsev347c5832018-12-24 06:06:17 +000049STATISTIC(NumLoopBlocksDeleted,
50 "Number of loop blocks deleted");
Max Kazantsevedabb9a2018-12-24 07:41:33 +000051STATISTIC(NumLoopExitsDeleted,
52 "Number of loop exiting edges deleted");
Max Kazantsevc04b5302018-11-20 05:43:32 +000053
54/// If \p BB is a switch or a conditional branch, but only one of its successors
55/// can be reached from this block in runtime, return this successor. Otherwise,
56/// return nullptr.
57static BasicBlock *getOnlyLiveSuccessor(BasicBlock *BB) {
58 Instruction *TI = BB->getTerminator();
59 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
60 if (BI->isUnconditional())
61 return nullptr;
62 if (BI->getSuccessor(0) == BI->getSuccessor(1))
63 return BI->getSuccessor(0);
64 ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());
65 if (!Cond)
66 return nullptr;
67 return Cond->isZero() ? BI->getSuccessor(1) : BI->getSuccessor(0);
68 }
69
70 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
71 auto *CI = dyn_cast<ConstantInt>(SI->getCondition());
72 if (!CI)
73 return nullptr;
74 for (auto Case : SI->cases())
75 if (Case.getCaseValue() == CI)
76 return Case.getCaseSuccessor();
77 return SI->getDefaultDest();
78 }
79
80 return nullptr;
81}
82
83/// Helper class that can turn branches and switches with constant conditions
84/// into unconditional branches.
85class ConstantTerminatorFoldingImpl {
86private:
87 Loop &L;
88 LoopInfo &LI;
89 DominatorTree &DT;
Max Kazantsev9cf417d2018-11-30 10:06:23 +000090 MemorySSAUpdater *MSSAU;
Max Kazantsevc04b5302018-11-20 05:43:32 +000091
Max Kazantseva523a212018-12-07 05:44:45 +000092 // Whether or not the current loop has irreducible CFG.
93 bool HasIrreducibleCFG = false;
Max Kazantsevc04b5302018-11-20 05:43:32 +000094 // Whether or not the current loop will still exist after terminator constant
95 // folding will be done. In theory, there are two ways how it can happen:
96 // 1. Loop's latch(es) become unreachable from loop header;
97 // 2. Loop's header becomes unreachable from method entry.
98 // In practice, the second situation is impossible because we only modify the
99 // current loop and its preheader and do not affect preheader's reachibility
100 // from any other block. So this variable set to true means that loop's latch
101 // has become unreachable from loop header.
102 bool DeleteCurrentLoop = false;
103
104 // The blocks of the original loop that will still be reachable from entry
105 // after the constant folding.
106 SmallPtrSet<BasicBlock *, 8> LiveLoopBlocks;
107 // The blocks of the original loop that will become unreachable from entry
108 // after the constant folding.
Max Kazantsev80e4b402018-12-28 06:08:51 +0000109 SmallVector<BasicBlock *, 8> DeadLoopBlocks;
Max Kazantsevc04b5302018-11-20 05:43:32 +0000110 // The exits of the original loop that will still be reachable from entry
111 // after the constant folding.
112 SmallPtrSet<BasicBlock *, 8> LiveExitBlocks;
113 // The exits of the original loop that will become unreachable from entry
114 // after the constant folding.
Max Kazantsev56a24432018-11-22 12:33:41 +0000115 SmallVector<BasicBlock *, 8> DeadExitBlocks;
Max Kazantsevc04b5302018-11-20 05:43:32 +0000116 // The blocks that will still be a part of the current loop after folding.
117 SmallPtrSet<BasicBlock *, 8> BlocksInLoopAfterFolding;
118 // The blocks that have terminators with constant condition that can be
119 // folded. Note: fold candidates should be in L but not in any of its
120 // subloops to avoid complex LI updates.
121 SmallVector<BasicBlock *, 8> FoldCandidates;
122
123 void dump() const {
124 dbgs() << "Constant terminator folding for loop " << L << "\n";
125 dbgs() << "After terminator constant-folding, the loop will";
126 if (!DeleteCurrentLoop)
127 dbgs() << " not";
128 dbgs() << " be destroyed\n";
Max Kazantsev56a24432018-11-22 12:33:41 +0000129 auto PrintOutVector = [&](const char *Message,
130 const SmallVectorImpl<BasicBlock *> &S) {
131 dbgs() << Message << "\n";
132 for (const BasicBlock *BB : S)
133 dbgs() << "\t" << BB->getName() << "\n";
134 };
Max Kazantsevc04b5302018-11-20 05:43:32 +0000135 auto PrintOutSet = [&](const char *Message,
136 const SmallPtrSetImpl<BasicBlock *> &S) {
137 dbgs() << Message << "\n";
138 for (const BasicBlock *BB : S)
139 dbgs() << "\t" << BB->getName() << "\n";
140 };
Max Kazantsev56a24432018-11-22 12:33:41 +0000141 PrintOutVector("Blocks in which we can constant-fold terminator:",
142 FoldCandidates);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000143 PrintOutSet("Live blocks from the original loop:", LiveLoopBlocks);
Max Kazantsev80e4b402018-12-28 06:08:51 +0000144 PrintOutVector("Dead blocks from the original loop:", DeadLoopBlocks);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000145 PrintOutSet("Live exit blocks:", LiveExitBlocks);
Max Kazantsev56a24432018-11-22 12:33:41 +0000146 PrintOutVector("Dead exit blocks:", DeadExitBlocks);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000147 if (!DeleteCurrentLoop)
148 PrintOutSet("The following blocks will still be part of the loop:",
149 BlocksInLoopAfterFolding);
150 }
151
Max Kazantseva523a212018-12-07 05:44:45 +0000152 /// Whether or not the current loop has irreducible CFG.
153 bool hasIrreducibleCFG(LoopBlocksDFS &DFS) {
154 assert(DFS.isComplete() && "DFS is expected to be finished");
155 // Index of a basic block in RPO traversal.
156 DenseMap<const BasicBlock *, unsigned> RPO;
157 unsigned Current = 0;
158 for (auto I = DFS.beginRPO(), E = DFS.endRPO(); I != E; ++I)
159 RPO[*I] = Current++;
160
161 for (auto I = DFS.beginRPO(), E = DFS.endRPO(); I != E; ++I) {
162 BasicBlock *BB = *I;
163 for (auto *Succ : successors(BB))
164 if (L.contains(Succ) && !LI.isLoopHeader(Succ) && RPO[BB] > RPO[Succ])
165 // If an edge goes from a block with greater order number into a block
166 // with lesses number, and it is not a loop backedge, then it can only
167 // be a part of irreducible non-loop cycle.
168 return true;
169 }
170 return false;
171 }
172
Max Kazantsevc04b5302018-11-20 05:43:32 +0000173 /// Fill all information about status of blocks and exits of the current loop
174 /// if constant folding of all branches will be done.
175 void analyze() {
176 LoopBlocksDFS DFS(&L);
177 DFS.perform(&LI);
178 assert(DFS.isComplete() && "DFS is expected to be finished");
179
Max Kazantseva523a212018-12-07 05:44:45 +0000180 // TODO: The algorithm below relies on both RPO and Postorder traversals.
181 // When the loop has only reducible CFG inside, then the invariant "all
182 // predecessors of X are processed before X in RPO" is preserved. However
183 // an irreducible loop can break this invariant (e.g. latch does not have to
184 // be the last block in the traversal in this case, and the algorithm relies
185 // on this). We can later decide to support such cases by altering the
186 // algorithms, but so far we just give up analyzing them.
187 if (hasIrreducibleCFG(DFS)) {
188 HasIrreducibleCFG = true;
189 return;
190 }
191
Max Kazantsevc04b5302018-11-20 05:43:32 +0000192 // Collect live and dead loop blocks and exits.
Max Kazantsevc04b5302018-11-20 05:43:32 +0000193 LiveLoopBlocks.insert(L.getHeader());
194 for (auto I = DFS.beginRPO(), E = DFS.endRPO(); I != E; ++I) {
195 BasicBlock *BB = *I;
196
197 // If a loop block wasn't marked as live so far, then it's dead.
198 if (!LiveLoopBlocks.count(BB)) {
Max Kazantsev80e4b402018-12-28 06:08:51 +0000199 DeadLoopBlocks.push_back(BB);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000200 continue;
201 }
202
203 BasicBlock *TheOnlySucc = getOnlyLiveSuccessor(BB);
204
205 // If a block has only one live successor, it's a candidate on constant
206 // folding. Only handle blocks from current loop: branches in child loops
207 // are skipped because if they can be folded, they should be folded during
208 // the processing of child loops.
209 if (TheOnlySucc && LI.getLoopFor(BB) == &L)
210 FoldCandidates.push_back(BB);
211
212 // Handle successors.
Max Kazantsevc04b5302018-11-20 05:43:32 +0000213 for (BasicBlock *Succ : successors(BB))
Max Kazantsevd9f59f82018-11-22 10:48:30 +0000214 if (!TheOnlySucc || TheOnlySucc == Succ) {
215 if (L.contains(Succ))
216 LiveLoopBlocks.insert(Succ);
217 else
218 LiveExitBlocks.insert(Succ);
219 }
Max Kazantsevc04b5302018-11-20 05:43:32 +0000220 }
221
222 // Sanity check: amount of dead and live loop blocks should match the total
223 // number of blocks in loop.
224 assert(L.getNumBlocks() == LiveLoopBlocks.size() + DeadLoopBlocks.size() &&
225 "Malformed block sets?");
226
227 // Now, all exit blocks that are not marked as live are dead.
Max Kazantsevd9f59f82018-11-22 10:48:30 +0000228 SmallVector<BasicBlock *, 8> ExitBlocks;
229 L.getExitBlocks(ExitBlocks);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000230 for (auto *ExitBlock : ExitBlocks)
231 if (!LiveExitBlocks.count(ExitBlock))
Max Kazantsev56a24432018-11-22 12:33:41 +0000232 DeadExitBlocks.push_back(ExitBlock);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000233
234 // Whether or not the edge From->To will still be present in graph after the
235 // folding.
236 auto IsEdgeLive = [&](BasicBlock *From, BasicBlock *To) {
237 if (!LiveLoopBlocks.count(From))
238 return false;
239 BasicBlock *TheOnlySucc = getOnlyLiveSuccessor(From);
240 return !TheOnlySucc || TheOnlySucc == To;
241 };
242
243 // The loop will not be destroyed if its latch is live.
244 DeleteCurrentLoop = !IsEdgeLive(L.getLoopLatch(), L.getHeader());
245
246 // If we are going to delete the current loop completely, no extra analysis
247 // is needed.
248 if (DeleteCurrentLoop)
249 return;
250
251 // Otherwise, we should check which blocks will still be a part of the
252 // current loop after the transform.
253 BlocksInLoopAfterFolding.insert(L.getLoopLatch());
254 // If the loop is live, then we should compute what blocks are still in
255 // loop after all branch folding has been done. A block is in loop if
256 // it has a live edge to another block that is in the loop; by definition,
257 // latch is in the loop.
258 auto BlockIsInLoop = [&](BasicBlock *BB) {
259 return any_of(successors(BB), [&](BasicBlock *Succ) {
260 return BlocksInLoopAfterFolding.count(Succ) && IsEdgeLive(BB, Succ);
261 });
262 };
263 for (auto I = DFS.beginPostorder(), E = DFS.endPostorder(); I != E; ++I) {
264 BasicBlock *BB = *I;
265 if (BlockIsInLoop(BB))
266 BlocksInLoopAfterFolding.insert(BB);
267 }
268
269 // Sanity check: header must be in loop.
270 assert(BlocksInLoopAfterFolding.count(L.getHeader()) &&
271 "Header not in loop?");
Max Kazantsevb565e602018-11-22 12:43:27 +0000272 assert(BlocksInLoopAfterFolding.size() <= LiveLoopBlocks.size() &&
273 "All blocks that stay in loop should be live!");
Max Kazantsevc04b5302018-11-20 05:43:32 +0000274 }
275
Max Kazantsevedabb9a2018-12-24 07:41:33 +0000276 /// We need to preserve static reachibility of all loop exit blocks (this is)
277 /// required by loop pass manager. In order to do it, we make the following
278 /// trick:
279 ///
280 /// preheader:
281 /// <preheader code>
282 /// br label %loop_header
283 ///
284 /// loop_header:
285 /// ...
286 /// br i1 false, label %dead_exit, label %loop_block
287 /// ...
288 ///
289 /// We cannot simply remove edge from the loop to dead exit because in this
290 /// case dead_exit (and its successors) may become unreachable. To avoid that,
291 /// we insert the following fictive preheader:
292 ///
293 /// preheader:
294 /// <preheader code>
295 /// switch i32 0, label %preheader-split,
296 /// [i32 1, label %dead_exit_1],
297 /// [i32 2, label %dead_exit_2],
298 /// ...
299 /// [i32 N, label %dead_exit_N],
300 ///
301 /// preheader-split:
302 /// br label %loop_header
303 ///
304 /// loop_header:
305 /// ...
306 /// br i1 false, label %dead_exit_N, label %loop_block
307 /// ...
308 ///
309 /// Doing so, we preserve static reachibility of all dead exits and can later
310 /// remove edges from the loop to these blocks.
311 void handleDeadExits() {
312 // If no dead exits, nothing to do.
313 if (DeadExitBlocks.empty())
314 return;
315
316 // Construct split preheader and the dummy switch to thread edges from it to
317 // dead exits.
318 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
319 BasicBlock *Preheader = L.getLoopPreheader();
320 BasicBlock *NewPreheader = Preheader->splitBasicBlock(
321 Preheader->getTerminator(),
322 Twine(Preheader->getName()).concat("-split"));
323 DTU.deleteEdge(Preheader, L.getHeader());
324 DTU.insertEdge(NewPreheader, L.getHeader());
325 DTU.insertEdge(Preheader, NewPreheader);
326 IRBuilder<> Builder(Preheader->getTerminator());
327 SwitchInst *DummySwitch =
328 Builder.CreateSwitch(Builder.getInt32(0), NewPreheader);
329 Preheader->getTerminator()->eraseFromParent();
330
331 unsigned DummyIdx = 1;
332 for (BasicBlock *BB : DeadExitBlocks) {
333 SmallVector<Instruction *, 4> DeadPhis;
334 for (auto &PN : BB->phis())
335 DeadPhis.push_back(&PN);
336
337 // Eliminate all Phis from dead exits.
338 for (Instruction *PN : DeadPhis) {
339 PN->replaceAllUsesWith(UndefValue::get(PN->getType()));
340 PN->eraseFromParent();
341 }
342 assert(DummyIdx != 0 && "Too many dead exits!");
343 DummySwitch->addCase(Builder.getInt32(DummyIdx++), BB);
344 DTU.insertEdge(Preheader, BB);
345 ++NumLoopExitsDeleted;
346 }
347
348 assert(L.getLoopPreheader() == NewPreheader && "Malformed CFG?");
349 if (Loop *OuterLoop = LI.getLoopFor(Preheader)) {
350 OuterLoop->addBasicBlockToLoop(NewPreheader, LI);
351
352 // When we break dead edges, the outer loop may become unreachable from
353 // the current loop. We need to fix loop info accordingly. For this, we
354 // find the most nested loop that still contains L and remove L from all
355 // loops that are inside of it.
356 Loop *StillReachable = nullptr;
357 for (BasicBlock *BB : LiveExitBlocks) {
358 Loop *BBL = LI.getLoopFor(BB);
359 if (BBL && BBL->contains(L.getHeader()))
360 if (!StillReachable ||
361 BBL->getLoopDepth() > StillReachable->getLoopDepth())
362 StillReachable = BBL;
363 }
364
365 // Okay, our loop is no longer in the outer loop (and maybe not in some of
366 // its parents as well). Make the fixup.
367 if (StillReachable != OuterLoop) {
368 LI.changeLoopFor(NewPreheader, StillReachable);
369 for (Loop *NotContaining = OuterLoop; NotContaining != StillReachable;
370 NotContaining = NotContaining->getParentLoop()) {
371 NotContaining->removeBlockFromLoop(NewPreheader);
372 for (auto *BB : L.blocks())
373 NotContaining->removeBlockFromLoop(BB);
374 }
375 OuterLoop->removeChildLoop(&L);
376 if (StillReachable)
377 StillReachable->addChildLoop(&L);
378 else
379 LI.addTopLevelLoop(&L);
380 }
381 }
382 }
383
Max Kazantsev347c5832018-12-24 06:06:17 +0000384 /// Delete loop blocks that have become unreachable after folding. Make all
385 /// relevant updates to DT and LI.
386 void deleteDeadLoopBlocks() {
387 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
Max Kazantsev80e4b402018-12-28 06:08:51 +0000388 if (MSSAU) {
389 SmallPtrSet<BasicBlock *, 8> DeadLoopBlocksSet(DeadLoopBlocks.begin(),
390 DeadLoopBlocks.end());
391 MSSAU->removeBlocks(DeadLoopBlocksSet);
392 }
Max Kazantsev347c5832018-12-24 06:06:17 +0000393 for (auto *BB : DeadLoopBlocks) {
394 assert(BB != L.getHeader() &&
395 "Header of the current loop cannot be dead!");
396 LLVM_DEBUG(dbgs() << "Deleting dead loop block " << BB->getName()
397 << "\n");
398 if (LI.isLoopHeader(BB)) {
399 assert(LI.getLoopFor(BB) != &L && "Attempt to remove current loop!");
400 LI.erase(LI.getLoopFor(BB));
401 }
402 LI.removeBlock(BB);
403 DeleteDeadBlock(BB, &DTU);
404 ++NumLoopBlocksDeleted;
405 }
406 }
407
Max Kazantsevc04b5302018-11-20 05:43:32 +0000408 /// Constant-fold terminators of blocks acculumated in FoldCandidates into the
409 /// unconditional branches.
410 void foldTerminators() {
411 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
412
413 for (BasicBlock *BB : FoldCandidates) {
414 assert(LI.getLoopFor(BB) == &L && "Should be a loop block!");
415 BasicBlock *TheOnlySucc = getOnlyLiveSuccessor(BB);
416 assert(TheOnlySucc && "Should have one live successor!");
417
418 LLVM_DEBUG(dbgs() << "Replacing terminator of " << BB->getName()
419 << " with an unconditional branch to the block "
420 << TheOnlySucc->getName() << "\n");
421
422 SmallPtrSet<BasicBlock *, 2> DeadSuccessors;
423 // Remove all BB's successors except for the live one.
Max Kazantsevc4e4d642018-11-27 06:17:21 +0000424 unsigned TheOnlySuccDuplicates = 0;
Max Kazantsevc04b5302018-11-20 05:43:32 +0000425 for (auto *Succ : successors(BB))
426 if (Succ != TheOnlySucc) {
427 DeadSuccessors.insert(Succ);
Max Kazantsevcb8e2402018-11-23 07:56:47 +0000428 // If our successor lies in a different loop, we don't want to remove
429 // the one-input Phi because it is a LCSSA Phi.
430 bool PreserveLCSSAPhi = !L.contains(Succ);
431 Succ->removePredecessor(BB, PreserveLCSSAPhi);
Max Kazantsev9cf417d2018-11-30 10:06:23 +0000432 if (MSSAU)
433 MSSAU->removeEdge(BB, Succ);
Max Kazantsevc4e4d642018-11-27 06:17:21 +0000434 } else
435 ++TheOnlySuccDuplicates;
436
437 assert(TheOnlySuccDuplicates > 0 && "Should be!");
438 // If TheOnlySucc was BB's successor more than once, after transform it
439 // will be its successor only once. Remove redundant inputs from
440 // TheOnlySucc's Phis.
441 bool PreserveLCSSAPhi = !L.contains(TheOnlySucc);
442 for (unsigned Dup = 1; Dup < TheOnlySuccDuplicates; ++Dup)
443 TheOnlySucc->removePredecessor(BB, PreserveLCSSAPhi);
Max Kazantsev9cf417d2018-11-30 10:06:23 +0000444 if (MSSAU && TheOnlySuccDuplicates > 1)
445 MSSAU->removeDuplicatePhiEdgesBetween(BB, TheOnlySucc);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000446
447 IRBuilder<> Builder(BB->getContext());
448 Instruction *Term = BB->getTerminator();
449 Builder.SetInsertPoint(Term);
450 Builder.CreateBr(TheOnlySucc);
451 Term->eraseFromParent();
452
453 for (auto *DeadSucc : DeadSuccessors)
454 DTU.deleteEdge(BB, DeadSucc);
455
456 ++NumTerminatorsFolded;
457 }
458 }
459
460public:
Max Kazantsev9cf417d2018-11-30 10:06:23 +0000461 ConstantTerminatorFoldingImpl(Loop &L, LoopInfo &LI, DominatorTree &DT,
462 MemorySSAUpdater *MSSAU)
463 : L(L), LI(LI), DT(DT), MSSAU(MSSAU) {}
Max Kazantsevc04b5302018-11-20 05:43:32 +0000464 bool run() {
465 assert(L.getLoopLatch() && "Should be single latch!");
466
467 // Collect all available information about status of blocks after constant
468 // folding.
469 analyze();
470
471 LLVM_DEBUG(dbgs() << "In function " << L.getHeader()->getParent()->getName()
472 << ": ");
473
Max Kazantseva523a212018-12-07 05:44:45 +0000474 if (HasIrreducibleCFG) {
475 LLVM_DEBUG(dbgs() << "Loops with irreducible CFG are not supported!\n");
476 return false;
477 }
478
Max Kazantsevc04b5302018-11-20 05:43:32 +0000479 // Nothing to constant-fold.
480 if (FoldCandidates.empty()) {
481 LLVM_DEBUG(
482 dbgs() << "No constant terminator folding candidates found in loop "
483 << L.getHeader()->getName() << "\n");
484 return false;
485 }
486
487 // TODO: Support deletion of the current loop.
488 if (DeleteCurrentLoop) {
489 LLVM_DEBUG(
490 dbgs()
491 << "Give up constant terminator folding in loop "
492 << L.getHeader()->getName()
493 << ": we don't currently support deletion of the current loop.\n");
494 return false;
495 }
496
Max Kazantsevc04b5302018-11-20 05:43:32 +0000497 // TODO: Support blocks that are not dead, but also not in loop after the
498 // folding.
Max Kazantsev347c5832018-12-24 06:06:17 +0000499 if (BlocksInLoopAfterFolding.size() + DeadLoopBlocks.size() !=
500 L.getNumBlocks()) {
Max Kazantsevc04b5302018-11-20 05:43:32 +0000501 LLVM_DEBUG(
502 dbgs() << "Give up constant terminator folding in loop "
503 << L.getHeader()->getName()
504 << ": we don't currently"
505 " support blocks that are not dead, but will stop "
506 "being a part of the loop after constant-folding.\n");
507 return false;
508 }
509
510 // Dump analysis results.
511 LLVM_DEBUG(dump());
512
513 LLVM_DEBUG(dbgs() << "Constant-folding " << FoldCandidates.size()
514 << " terminators in loop " << L.getHeader()->getName()
515 << "\n");
516
517 // Make the actual transforms.
Max Kazantsevedabb9a2018-12-24 07:41:33 +0000518 handleDeadExits();
Max Kazantsevc04b5302018-11-20 05:43:32 +0000519 foldTerminators();
520
Max Kazantsev347c5832018-12-24 06:06:17 +0000521 if (!DeadLoopBlocks.empty()) {
522 LLVM_DEBUG(dbgs() << "Deleting " << DeadLoopBlocks.size()
523 << " dead blocks in loop " << L.getHeader()->getName()
524 << "\n");
525 deleteDeadLoopBlocks();
526 }
527
Max Kazantsevc04b5302018-11-20 05:43:32 +0000528#ifndef NDEBUG
529 // Make sure that we have preserved all data structures after the transform.
530 DT.verify();
531 assert(DT.isReachableFromEntry(L.getHeader()));
532 LI.verify(DT);
533#endif
534
535 return true;
536 }
537};
538
539/// Turn branches and switches with known constant conditions into unconditional
540/// branches.
Max Kazantsev9cf417d2018-11-30 10:06:23 +0000541static bool constantFoldTerminators(Loop &L, DominatorTree &DT, LoopInfo &LI,
542 MemorySSAUpdater *MSSAU) {
Max Kazantseve1c2dc22018-11-23 09:14:53 +0000543 if (!EnableTermFolding)
544 return false;
545
Max Kazantsevc04b5302018-11-20 05:43:32 +0000546 // To keep things simple, only process loops with single latch. We
547 // canonicalize most loops to this form. We can support multi-latch if needed.
548 if (!L.getLoopLatch())
549 return false;
550
Max Kazantsev9cf417d2018-11-30 10:06:23 +0000551 ConstantTerminatorFoldingImpl BranchFolder(L, LI, DT, MSSAU);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000552 return BranchFolder.run();
553}
554
Max Kazantsev46955b52018-11-01 09:42:50 +0000555static bool mergeBlocksIntoPredecessors(Loop &L, DominatorTree &DT,
556 LoopInfo &LI, MemorySSAUpdater *MSSAU) {
Fiona Glaserb417d462016-01-29 22:35:36 +0000557 bool Changed = false;
Chijun Sima21a8b602018-08-03 05:08:17 +0000558 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
Fiona Glaserb417d462016-01-29 22:35:36 +0000559 // Copy blocks into a temporary array to avoid iterator invalidation issues
560 // as we remove them.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000561 SmallVector<WeakTrackingVH, 16> Blocks(L.blocks());
Fiona Glaserb417d462016-01-29 22:35:36 +0000562
563 for (auto &Block : Blocks) {
564 // Attempt to merge blocks in the trivial case. Don't modify blocks which
565 // belong to other loops.
Fiona Glaser36e82302016-01-29 23:12:52 +0000566 BasicBlock *Succ = cast_or_null<BasicBlock>(Block);
Fiona Glaserb417d462016-01-29 22:35:36 +0000567 if (!Succ)
568 continue;
569
570 BasicBlock *Pred = Succ->getSinglePredecessor();
Justin Bognerab6a5132016-05-03 21:47:32 +0000571 if (!Pred || !Pred->getSingleSuccessor() || LI.getLoopFor(Pred) != &L)
Fiona Glaserb417d462016-01-29 22:35:36 +0000572 continue;
573
Alina Sbirleadfd14ad2018-06-20 22:01:04 +0000574 // Merge Succ into Pred and delete it.
Alina Sbirlea8b83d682018-08-22 20:10:21 +0000575 MergeBlockIntoPredecessor(Succ, &DTU, &LI, MSSAU);
David Greene6a9c242018-06-19 09:43:36 +0000576
Fiona Glaserb417d462016-01-29 22:35:36 +0000577 Changed = true;
578 }
579
580 return Changed;
581}
582
Max Kazantsev46955b52018-11-01 09:42:50 +0000583static bool simplifyLoopCFG(Loop &L, DominatorTree &DT, LoopInfo &LI,
584 ScalarEvolution &SE, MemorySSAUpdater *MSSAU) {
585 bool Changed = false;
586
Max Kazantsevc04b5302018-11-20 05:43:32 +0000587 // Constant-fold terminators with known constant conditions.
Max Kazantsev9cf417d2018-11-30 10:06:23 +0000588 Changed |= constantFoldTerminators(L, DT, LI, MSSAU);
Max Kazantsevc04b5302018-11-20 05:43:32 +0000589
Max Kazantsev46955b52018-11-01 09:42:50 +0000590 // Eliminate unconditional branches by merging blocks into their predecessors.
591 Changed |= mergeBlocksIntoPredecessors(L, DT, LI, MSSAU);
592
593 if (Changed)
594 SE.forgetTopmostLoop(&L);
595
596 return Changed;
597}
598
Chandler Carruth410eaeb2017-01-11 06:23:21 +0000599PreservedAnalyses LoopSimplifyCFGPass::run(Loop &L, LoopAnalysisManager &AM,
600 LoopStandardAnalysisResults &AR,
601 LPMUpdater &) {
Alina Sbirlea8b83d682018-08-22 20:10:21 +0000602 Optional<MemorySSAUpdater> MSSAU;
603 if (EnableMSSALoopDependency && AR.MSSA)
604 MSSAU = MemorySSAUpdater(AR.MSSA);
605 if (!simplifyLoopCFG(L, AR.DT, AR.LI, AR.SE,
606 MSSAU.hasValue() ? MSSAU.getPointer() : nullptr))
Justin Bognerab6a5132016-05-03 21:47:32 +0000607 return PreservedAnalyses::all();
Chandler Carruthca68a3e2017-01-15 06:32:49 +0000608
Justin Bognerab6a5132016-05-03 21:47:32 +0000609 return getLoopPassPreservedAnalyses();
610}
611
612namespace {
613class LoopSimplifyCFGLegacyPass : public LoopPass {
614public:
615 static char ID; // Pass ID, replacement for typeid
616 LoopSimplifyCFGLegacyPass() : LoopPass(ID) {
617 initializeLoopSimplifyCFGLegacyPassPass(*PassRegistry::getPassRegistry());
618 }
619
620 bool runOnLoop(Loop *L, LPPassManager &) override {
621 if (skipLoop(L))
622 return false;
623
624 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
625 LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
David Greene6a9c242018-06-19 09:43:36 +0000626 ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
Alina Sbirlea8b83d682018-08-22 20:10:21 +0000627 Optional<MemorySSAUpdater> MSSAU;
628 if (EnableMSSALoopDependency) {
629 MemorySSA *MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA();
630 MSSAU = MemorySSAUpdater(MSSA);
631 if (VerifyMemorySSA)
632 MSSA->verifyMemorySSA();
633 }
634 return simplifyLoopCFG(*L, DT, LI, SE,
635 MSSAU.hasValue() ? MSSAU.getPointer() : nullptr);
Justin Bognerab6a5132016-05-03 21:47:32 +0000636 }
637
638 void getAnalysisUsage(AnalysisUsage &AU) const override {
Alina Sbirlea8b83d682018-08-22 20:10:21 +0000639 if (EnableMSSALoopDependency) {
640 AU.addRequired<MemorySSAWrapperPass>();
641 AU.addPreserved<MemorySSAWrapperPass>();
642 }
Chandler Carruth49c22192016-05-12 22:19:39 +0000643 AU.addPreserved<DependenceAnalysisWrapperPass>();
Justin Bognerab6a5132016-05-03 21:47:32 +0000644 getLoopAnalysisUsage(AU);
645 }
646};
647}
648
649char LoopSimplifyCFGLegacyPass::ID = 0;
650INITIALIZE_PASS_BEGIN(LoopSimplifyCFGLegacyPass, "loop-simplifycfg",
651 "Simplify loop CFG", false, false)
652INITIALIZE_PASS_DEPENDENCY(LoopPass)
Alina Sbirlea8b83d682018-08-22 20:10:21 +0000653INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
Justin Bognerab6a5132016-05-03 21:47:32 +0000654INITIALIZE_PASS_END(LoopSimplifyCFGLegacyPass, "loop-simplifycfg",
655 "Simplify loop CFG", false, false)
656
657Pass *llvm::createLoopSimplifyCFGPass() {
658 return new LoopSimplifyCFGLegacyPass();
Fiona Glaserb417d462016-01-29 22:35:36 +0000659}