blob: 95368d22c4bdbf3ca64a3a21417a0b974f8d9589 [file] [log] [blame]
David Greenb0aa36f2018-03-29 08:48:15 +00001//===----------------- LoopRotationUtils.cpp -----------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file provides utilities to convert a loop into a loop with bottom test.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/LoopRotationUtils.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/BasicAliasAnalysis.h"
19#include "llvm/Analysis/CodeMetrics.h"
20#include "llvm/Analysis/GlobalsModRef.h"
21#include "llvm/Analysis/InstructionSimplify.h"
22#include "llvm/Analysis/LoopPass.h"
23#include "llvm/Analysis/ScalarEvolution.h"
24#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
25#include "llvm/Analysis/TargetTransformInfo.h"
26#include "llvm/Analysis/Utils/Local.h"
27#include "llvm/Analysis/ValueTracking.h"
28#include "llvm/IR/CFG.h"
29#include "llvm/IR/DebugInfoMetadata.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Module.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/raw_ostream.h"
37#include "llvm/Transforms/Scalar.h"
38#include "llvm/Transforms/Scalar/LoopPassManager.h"
39#include "llvm/Transforms/Utils/BasicBlockUtils.h"
40#include "llvm/Transforms/Utils/LoopUtils.h"
41#include "llvm/Transforms/Utils/SSAUpdater.h"
42#include "llvm/Transforms/Utils/ValueMapper.h"
43using namespace llvm;
44
45#define DEBUG_TYPE "loop-rotate"
46
47STATISTIC(NumRotated, "Number of loops rotated");
48
49namespace {
50/// A simple loop rotation transformation.
51class LoopRotate {
52 const unsigned MaxHeaderSize;
53 LoopInfo *LI;
54 const TargetTransformInfo *TTI;
55 AssumptionCache *AC;
56 DominatorTree *DT;
57 ScalarEvolution *SE;
58 const SimplifyQuery &SQ;
59
60public:
61 LoopRotate(unsigned MaxHeaderSize, LoopInfo *LI,
62 const TargetTransformInfo *TTI, AssumptionCache *AC,
63 DominatorTree *DT, ScalarEvolution *SE, const SimplifyQuery &SQ)
64 : MaxHeaderSize(MaxHeaderSize), LI(LI), TTI(TTI), AC(AC), DT(DT), SE(SE),
65 SQ(SQ) {}
66 bool processLoop(Loop *L);
67
68private:
69 bool rotateLoop(Loop *L, bool SimplifiedLatch);
70 bool simplifyLoopLatch(Loop *L);
71};
72} // end anonymous namespace
73
74/// RewriteUsesOfClonedInstructions - We just cloned the instructions from the
75/// old header into the preheader. If there were uses of the values produced by
76/// these instruction that were outside of the loop, we have to insert PHI nodes
77/// to merge the two values. Do this now.
78static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader,
79 BasicBlock *OrigPreheader,
80 ValueToValueMapTy &ValueMap,
81 SmallVectorImpl<PHINode*> *InsertedPHIs) {
82 // Remove PHI node entries that are no longer live.
83 BasicBlock::iterator I, E = OrigHeader->end();
84 for (I = OrigHeader->begin(); PHINode *PN = dyn_cast<PHINode>(I); ++I)
85 PN->removeIncomingValue(PN->getBasicBlockIndex(OrigPreheader));
86
87 // Now fix up users of the instructions in OrigHeader, inserting PHI nodes
88 // as necessary.
89 SSAUpdater SSA(InsertedPHIs);
90 for (I = OrigHeader->begin(); I != E; ++I) {
91 Value *OrigHeaderVal = &*I;
92
93 // If there are no uses of the value (e.g. because it returns void), there
94 // is nothing to rewrite.
95 if (OrigHeaderVal->use_empty())
96 continue;
97
98 Value *OrigPreHeaderVal = ValueMap.lookup(OrigHeaderVal);
99
100 // The value now exits in two versions: the initial value in the preheader
101 // and the loop "next" value in the original header.
102 SSA.Initialize(OrigHeaderVal->getType(), OrigHeaderVal->getName());
103 SSA.AddAvailableValue(OrigHeader, OrigHeaderVal);
104 SSA.AddAvailableValue(OrigPreheader, OrigPreHeaderVal);
105
106 // Visit each use of the OrigHeader instruction.
107 for (Value::use_iterator UI = OrigHeaderVal->use_begin(),
108 UE = OrigHeaderVal->use_end();
109 UI != UE;) {
110 // Grab the use before incrementing the iterator.
111 Use &U = *UI;
112
113 // Increment the iterator before removing the use from the list.
114 ++UI;
115
116 // SSAUpdater can't handle a non-PHI use in the same block as an
117 // earlier def. We can easily handle those cases manually.
118 Instruction *UserInst = cast<Instruction>(U.getUser());
119 if (!isa<PHINode>(UserInst)) {
120 BasicBlock *UserBB = UserInst->getParent();
121
122 // The original users in the OrigHeader are already using the
123 // original definitions.
124 if (UserBB == OrigHeader)
125 continue;
126
127 // Users in the OrigPreHeader need to use the value to which the
128 // original definitions are mapped.
129 if (UserBB == OrigPreheader) {
130 U = OrigPreHeaderVal;
131 continue;
132 }
133 }
134
135 // Anything else can be handled by SSAUpdater.
136 SSA.RewriteUse(U);
137 }
138
139 // Replace MetadataAsValue(ValueAsMetadata(OrigHeaderVal)) uses in debug
140 // intrinsics.
141 SmallVector<DbgValueInst *, 1> DbgValues;
142 llvm::findDbgValues(DbgValues, OrigHeaderVal);
143 for (auto &DbgValue : DbgValues) {
144 // The original users in the OrigHeader are already using the original
145 // definitions.
146 BasicBlock *UserBB = DbgValue->getParent();
147 if (UserBB == OrigHeader)
148 continue;
149
150 // Users in the OrigPreHeader need to use the value to which the
151 // original definitions are mapped and anything else can be handled by
152 // the SSAUpdater. To avoid adding PHINodes, check if the value is
153 // available in UserBB, if not substitute undef.
154 Value *NewVal;
155 if (UserBB == OrigPreheader)
156 NewVal = OrigPreHeaderVal;
157 else if (SSA.HasValueForBlock(UserBB))
158 NewVal = SSA.GetValueInMiddleOfBlock(UserBB);
159 else
160 NewVal = UndefValue::get(OrigHeaderVal->getType());
161 DbgValue->setOperand(0,
162 MetadataAsValue::get(OrigHeaderVal->getContext(),
163 ValueAsMetadata::get(NewVal)));
164 }
165 }
166}
167
168/// Rotate loop LP. Return true if the loop is rotated.
169///
170/// \param SimplifiedLatch is true if the latch was just folded into the final
171/// loop exit. In this case we may want to rotate even though the new latch is
172/// now an exiting branch. This rotation would have happened had the latch not
173/// been simplified. However, if SimplifiedLatch is false, then we avoid
174/// rotating loops in which the latch exits to avoid excessive or endless
175/// rotation. LoopRotate should be repeatable and converge to a canonical
176/// form. This property is satisfied because simplifying the loop latch can only
177/// happen once across multiple invocations of the LoopRotate pass.
178bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
179 // If the loop has only one block then there is not much to rotate.
180 if (L->getBlocks().size() == 1)
181 return false;
182
183 BasicBlock *OrigHeader = L->getHeader();
184 BasicBlock *OrigLatch = L->getLoopLatch();
185
186 BranchInst *BI = dyn_cast<BranchInst>(OrigHeader->getTerminator());
187 if (!BI || BI->isUnconditional())
188 return false;
189
190 // If the loop header is not one of the loop exiting blocks then
191 // either this loop is already rotated or it is not
192 // suitable for loop rotation transformations.
193 if (!L->isLoopExiting(OrigHeader))
194 return false;
195
196 // If the loop latch already contains a branch that leaves the loop then the
197 // loop is already rotated.
198 if (!OrigLatch)
199 return false;
200
201 // Rotate if either the loop latch does *not* exit the loop, or if the loop
202 // latch was just simplified.
203 if (L->isLoopExiting(OrigLatch) && !SimplifiedLatch)
204 return false;
205
206 // Check size of original header and reject loop if it is very big or we can't
207 // duplicate blocks inside it.
208 {
209 SmallPtrSet<const Value *, 32> EphValues;
210 CodeMetrics::collectEphemeralValues(L, AC, EphValues);
211
212 CodeMetrics Metrics;
213 Metrics.analyzeBasicBlock(OrigHeader, *TTI, EphValues);
214 if (Metrics.notDuplicatable) {
215 DEBUG(dbgs() << "LoopRotation: NOT rotating - contains non-duplicatable"
216 << " instructions: ";
217 L->dump());
218 return false;
219 }
220 if (Metrics.convergent) {
221 DEBUG(dbgs() << "LoopRotation: NOT rotating - contains convergent "
222 "instructions: ";
223 L->dump());
224 return false;
225 }
226 if (Metrics.NumInsts > MaxHeaderSize)
227 return false;
228 }
229
230 // Now, this loop is suitable for rotation.
231 BasicBlock *OrigPreheader = L->getLoopPreheader();
232
233 // If the loop could not be converted to canonical form, it must have an
234 // indirectbr in it, just give up.
235 if (!OrigPreheader || !L->hasDedicatedExits())
236 return false;
237
238 // Anything ScalarEvolution may know about this loop or the PHI nodes
239 // in its header will soon be invalidated.
240 if (SE)
241 SE->forgetLoop(L);
242
243 DEBUG(dbgs() << "LoopRotation: rotating "; L->dump());
244
245 // Find new Loop header. NewHeader is a Header's one and only successor
246 // that is inside loop. Header's other successor is outside the
247 // loop. Otherwise loop is not suitable for rotation.
248 BasicBlock *Exit = BI->getSuccessor(0);
249 BasicBlock *NewHeader = BI->getSuccessor(1);
250 if (L->contains(Exit))
251 std::swap(Exit, NewHeader);
252 assert(NewHeader && "Unable to determine new loop header");
253 assert(L->contains(NewHeader) && !L->contains(Exit) &&
254 "Unable to determine loop header and exit blocks");
255
256 // This code assumes that the new header has exactly one predecessor.
257 // Remove any single-entry PHI nodes in it.
258 assert(NewHeader->getSinglePredecessor() &&
259 "New header doesn't have one pred!");
260 FoldSingleEntryPHINodes(NewHeader);
261
262 // Begin by walking OrigHeader and populating ValueMap with an entry for
263 // each Instruction.
264 BasicBlock::iterator I = OrigHeader->begin(), E = OrigHeader->end();
265 ValueToValueMapTy ValueMap;
266
267 // For PHI nodes, the value available in OldPreHeader is just the
268 // incoming value from OldPreHeader.
269 for (; PHINode *PN = dyn_cast<PHINode>(I); ++I)
270 ValueMap[PN] = PN->getIncomingValueForBlock(OrigPreheader);
271
272 // For the rest of the instructions, either hoist to the OrigPreheader if
273 // possible or create a clone in the OldPreHeader if not.
274 TerminatorInst *LoopEntryBranch = OrigPreheader->getTerminator();
275
276 // Record all debug intrinsics preceding LoopEntryBranch to avoid duplication.
277 using DbgIntrinsicHash =
278 std::pair<std::pair<Value *, DILocalVariable *>, DIExpression *>;
279 auto makeHash = [](DbgInfoIntrinsic *D) -> DbgIntrinsicHash {
280 return {{D->getVariableLocation(), D->getVariable()}, D->getExpression()};
281 };
282 SmallDenseSet<DbgIntrinsicHash, 8> DbgIntrinsics;
283 for (auto I = std::next(OrigPreheader->rbegin()), E = OrigPreheader->rend();
284 I != E; ++I) {
285 if (auto *DII = dyn_cast<DbgInfoIntrinsic>(&*I))
286 DbgIntrinsics.insert(makeHash(DII));
287 else
288 break;
289 }
290
291 while (I != E) {
292 Instruction *Inst = &*I++;
293
294 // If the instruction's operands are invariant and it doesn't read or write
295 // memory, then it is safe to hoist. Doing this doesn't change the order of
296 // execution in the preheader, but does prevent the instruction from
297 // executing in each iteration of the loop. This means it is safe to hoist
298 // something that might trap, but isn't safe to hoist something that reads
299 // memory (without proving that the loop doesn't write).
300 if (L->hasLoopInvariantOperands(Inst) && !Inst->mayReadFromMemory() &&
301 !Inst->mayWriteToMemory() && !isa<TerminatorInst>(Inst) &&
302 !isa<DbgInfoIntrinsic>(Inst) && !isa<AllocaInst>(Inst)) {
303 Inst->moveBefore(LoopEntryBranch);
304 continue;
305 }
306
307 // Otherwise, create a duplicate of the instruction.
308 Instruction *C = Inst->clone();
309
310 // Eagerly remap the operands of the instruction.
311 RemapInstruction(C, ValueMap,
312 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
313
314 // Avoid inserting the same intrinsic twice.
315 if (auto *DII = dyn_cast<DbgInfoIntrinsic>(C))
316 if (DbgIntrinsics.count(makeHash(DII))) {
317 C->deleteValue();
318 continue;
319 }
320
321 // With the operands remapped, see if the instruction constant folds or is
322 // otherwise simplifyable. This commonly occurs because the entry from PHI
323 // nodes allows icmps and other instructions to fold.
324 Value *V = SimplifyInstruction(C, SQ);
325 if (V && LI->replacementPreservesLCSSAForm(C, V)) {
326 // If so, then delete the temporary instruction and stick the folded value
327 // in the map.
328 ValueMap[Inst] = V;
329 if (!C->mayHaveSideEffects()) {
330 C->deleteValue();
331 C = nullptr;
332 }
333 } else {
334 ValueMap[Inst] = C;
335 }
336 if (C) {
337 // Otherwise, stick the new instruction into the new block!
338 C->setName(Inst->getName());
339 C->insertBefore(LoopEntryBranch);
340
341 if (auto *II = dyn_cast<IntrinsicInst>(C))
342 if (II->getIntrinsicID() == Intrinsic::assume)
343 AC->registerAssumption(II);
344 }
345 }
346
347 // Along with all the other instructions, we just cloned OrigHeader's
348 // terminator into OrigPreHeader. Fix up the PHI nodes in each of OrigHeader's
349 // successors by duplicating their incoming values for OrigHeader.
350 TerminatorInst *TI = OrigHeader->getTerminator();
351 for (BasicBlock *SuccBB : TI->successors())
352 for (BasicBlock::iterator BI = SuccBB->begin();
353 PHINode *PN = dyn_cast<PHINode>(BI); ++BI)
354 PN->addIncoming(PN->getIncomingValueForBlock(OrigHeader), OrigPreheader);
355
356 // Now that OrigPreHeader has a clone of OrigHeader's terminator, remove
357 // OrigPreHeader's old terminator (the original branch into the loop), and
358 // remove the corresponding incoming values from the PHI nodes in OrigHeader.
359 LoopEntryBranch->eraseFromParent();
360
361
362 SmallVector<PHINode*, 2> InsertedPHIs;
363 // If there were any uses of instructions in the duplicated block outside the
364 // loop, update them, inserting PHI nodes as required
365 RewriteUsesOfClonedInstructions(OrigHeader, OrigPreheader, ValueMap,
366 &InsertedPHIs);
367
368 // Attach dbg.value intrinsics to the new phis if that phi uses a value that
369 // previously had debug metadata attached. This keeps the debug info
370 // up-to-date in the loop body.
371 if (!InsertedPHIs.empty())
372 insertDebugValuesForPHIs(OrigHeader, InsertedPHIs);
373
374 // NewHeader is now the header of the loop.
375 L->moveToHeader(NewHeader);
376 assert(L->getHeader() == NewHeader && "Latch block is our new header");
377
378 // Inform DT about changes to the CFG.
379 if (DT) {
380 // The OrigPreheader branches to the NewHeader and Exit now. Then, inform
381 // the DT about the removed edge to the OrigHeader (that got removed).
382 SmallVector<DominatorTree::UpdateType, 3> Updates;
383 Updates.push_back({DominatorTree::Insert, OrigPreheader, Exit});
384 Updates.push_back({DominatorTree::Insert, OrigPreheader, NewHeader});
385 Updates.push_back({DominatorTree::Delete, OrigPreheader, OrigHeader});
386 DT->applyUpdates(Updates);
387 }
388
389 // At this point, we've finished our major CFG changes. As part of cloning
390 // the loop into the preheader we've simplified instructions and the
391 // duplicated conditional branch may now be branching on a constant. If it is
392 // branching on a constant and if that constant means that we enter the loop,
393 // then we fold away the cond branch to an uncond branch. This simplifies the
394 // loop in cases important for nested loops, and it also means we don't have
395 // to split as many edges.
396 BranchInst *PHBI = cast<BranchInst>(OrigPreheader->getTerminator());
397 assert(PHBI->isConditional() && "Should be clone of BI condbr!");
398 if (!isa<ConstantInt>(PHBI->getCondition()) ||
399 PHBI->getSuccessor(cast<ConstantInt>(PHBI->getCondition())->isZero()) !=
400 NewHeader) {
401 // The conditional branch can't be folded, handle the general case.
402 // Split edges as necessary to preserve LoopSimplify form.
403
404 // Right now OrigPreHeader has two successors, NewHeader and ExitBlock, and
405 // thus is not a preheader anymore.
406 // Split the edge to form a real preheader.
407 BasicBlock *NewPH = SplitCriticalEdge(
408 OrigPreheader, NewHeader,
409 CriticalEdgeSplittingOptions(DT, LI).setPreserveLCSSA());
410 NewPH->setName(NewHeader->getName() + ".lr.ph");
411
412 // Preserve canonical loop form, which means that 'Exit' should have only
413 // one predecessor. Note that Exit could be an exit block for multiple
414 // nested loops, causing both of the edges to now be critical and need to
415 // be split.
416 SmallVector<BasicBlock *, 4> ExitPreds(pred_begin(Exit), pred_end(Exit));
417 bool SplitLatchEdge = false;
418 for (BasicBlock *ExitPred : ExitPreds) {
419 // We only need to split loop exit edges.
420 Loop *PredLoop = LI->getLoopFor(ExitPred);
421 if (!PredLoop || PredLoop->contains(Exit))
422 continue;
423 if (isa<IndirectBrInst>(ExitPred->getTerminator()))
424 continue;
425 SplitLatchEdge |= L->getLoopLatch() == ExitPred;
426 BasicBlock *ExitSplit = SplitCriticalEdge(
427 ExitPred, Exit,
428 CriticalEdgeSplittingOptions(DT, LI).setPreserveLCSSA());
429 ExitSplit->moveBefore(Exit);
430 }
431 assert(SplitLatchEdge &&
432 "Despite splitting all preds, failed to split latch exit?");
433 } else {
434 // We can fold the conditional branch in the preheader, this makes things
435 // simpler. The first step is to remove the extra edge to the Exit block.
436 Exit->removePredecessor(OrigPreheader, true /*preserve LCSSA*/);
437 BranchInst *NewBI = BranchInst::Create(NewHeader, PHBI);
438 NewBI->setDebugLoc(PHBI->getDebugLoc());
439 PHBI->eraseFromParent();
440
441 // With our CFG finalized, update DomTree if it is available.
442 if (DT) DT->deleteEdge(OrigPreheader, Exit);
443 }
444
445 assert(L->getLoopPreheader() && "Invalid loop preheader after loop rotation");
446 assert(L->getLoopLatch() && "Invalid loop latch after loop rotation");
447
448 // Now that the CFG and DomTree are in a consistent state again, try to merge
449 // the OrigHeader block into OrigLatch. This will succeed if they are
450 // connected by an unconditional branch. This is just a cleanup so the
451 // emitted code isn't too gross in this common case.
452 MergeBlockIntoPredecessor(OrigHeader, DT, LI);
453
454 DEBUG(dbgs() << "LoopRotation: into "; L->dump());
455
456 ++NumRotated;
457 return true;
458}
459
460/// Determine whether the instructions in this range may be safely and cheaply
461/// speculated. This is not an important enough situation to develop complex
462/// heuristics. We handle a single arithmetic instruction along with any type
463/// conversions.
464static bool shouldSpeculateInstrs(BasicBlock::iterator Begin,
465 BasicBlock::iterator End, Loop *L) {
466 bool seenIncrement = false;
467 bool MultiExitLoop = false;
468
469 if (!L->getExitingBlock())
470 MultiExitLoop = true;
471
472 for (BasicBlock::iterator I = Begin; I != End; ++I) {
473
474 if (!isSafeToSpeculativelyExecute(&*I))
475 return false;
476
477 if (isa<DbgInfoIntrinsic>(I))
478 continue;
479
480 switch (I->getOpcode()) {
481 default:
482 return false;
483 case Instruction::GetElementPtr:
484 // GEPs are cheap if all indices are constant.
485 if (!cast<GEPOperator>(I)->hasAllConstantIndices())
486 return false;
487 // fall-thru to increment case
488 LLVM_FALLTHROUGH;
489 case Instruction::Add:
490 case Instruction::Sub:
491 case Instruction::And:
492 case Instruction::Or:
493 case Instruction::Xor:
494 case Instruction::Shl:
495 case Instruction::LShr:
496 case Instruction::AShr: {
497 Value *IVOpnd =
498 !isa<Constant>(I->getOperand(0))
499 ? I->getOperand(0)
500 : !isa<Constant>(I->getOperand(1)) ? I->getOperand(1) : nullptr;
501 if (!IVOpnd)
502 return false;
503
504 // If increment operand is used outside of the loop, this speculation
505 // could cause extra live range interference.
506 if (MultiExitLoop) {
507 for (User *UseI : IVOpnd->users()) {
508 auto *UserInst = cast<Instruction>(UseI);
509 if (!L->contains(UserInst))
510 return false;
511 }
512 }
513
514 if (seenIncrement)
515 return false;
516 seenIncrement = true;
517 break;
518 }
519 case Instruction::Trunc:
520 case Instruction::ZExt:
521 case Instruction::SExt:
522 // ignore type conversions
523 break;
524 }
525 }
526 return true;
527}
528
529/// Fold the loop tail into the loop exit by speculating the loop tail
530/// instructions. Typically, this is a single post-increment. In the case of a
531/// simple 2-block loop, hoisting the increment can be much better than
532/// duplicating the entire loop header. In the case of loops with early exits,
533/// rotation will not work anyway, but simplifyLoopLatch will put the loop in
534/// canonical form so downstream passes can handle it.
535///
536/// I don't believe this invalidates SCEV.
537bool LoopRotate::simplifyLoopLatch(Loop *L) {
538 BasicBlock *Latch = L->getLoopLatch();
539 if (!Latch || Latch->hasAddressTaken())
540 return false;
541
542 BranchInst *Jmp = dyn_cast<BranchInst>(Latch->getTerminator());
543 if (!Jmp || !Jmp->isUnconditional())
544 return false;
545
546 BasicBlock *LastExit = Latch->getSinglePredecessor();
547 if (!LastExit || !L->isLoopExiting(LastExit))
548 return false;
549
550 BranchInst *BI = dyn_cast<BranchInst>(LastExit->getTerminator());
551 if (!BI)
552 return false;
553
554 if (!shouldSpeculateInstrs(Latch->begin(), Jmp->getIterator(), L))
555 return false;
556
557 DEBUG(dbgs() << "Folding loop latch " << Latch->getName() << " into "
558 << LastExit->getName() << "\n");
559
560 // Hoist the instructions from Latch into LastExit.
561 LastExit->getInstList().splice(BI->getIterator(), Latch->getInstList(),
562 Latch->begin(), Jmp->getIterator());
563
564 unsigned FallThruPath = BI->getSuccessor(0) == Latch ? 0 : 1;
565 BasicBlock *Header = Jmp->getSuccessor(0);
566 assert(Header == L->getHeader() && "expected a backward branch");
567
568 // Remove Latch from the CFG so that LastExit becomes the new Latch.
569 BI->setSuccessor(FallThruPath, Header);
570 Latch->replaceSuccessorsPhiUsesWith(LastExit);
571 Jmp->eraseFromParent();
572
573 // Nuke the Latch block.
574 assert(Latch->empty() && "unable to evacuate Latch");
575 LI->removeBlock(Latch);
576 if (DT)
577 DT->eraseNode(Latch);
578 Latch->eraseFromParent();
579 return true;
580}
581
582/// Rotate \c L, and return true if any modification was made.
583bool LoopRotate::processLoop(Loop *L) {
584 // Save the loop metadata.
585 MDNode *LoopMD = L->getLoopID();
586
587 // Simplify the loop latch before attempting to rotate the header
588 // upward. Rotation may not be needed if the loop tail can be folded into the
589 // loop exit.
590 bool SimplifiedLatch = simplifyLoopLatch(L);
591
592 bool MadeChange = rotateLoop(L, SimplifiedLatch);
593 assert((!MadeChange || L->isLoopExiting(L->getLoopLatch())) &&
594 "Loop latch should be exiting after loop-rotate.");
595
596 // Restore the loop metadata.
597 // NB! We presume LoopRotation DOESN'T ADD its own metadata.
598 if ((MadeChange || SimplifiedLatch) && LoopMD)
599 L->setLoopID(LoopMD);
600
601 return MadeChange || SimplifiedLatch;
602}
603
604
605/// The utility to convert a loop into a loop with bottom test.
606bool llvm::LoopRotation(Loop *L, unsigned MaxHeaderSize, LoopInfo *LI,
607 const TargetTransformInfo *TTI, AssumptionCache *AC,
608 DominatorTree *DT, ScalarEvolution *SE,
609 const SimplifyQuery &SQ) {
610 LoopRotate LR(MaxHeaderSize, LI, TTI, AC, DT, SE, SQ);
611
612 return LR.processLoop(L);
613}