blob: aad82a442aed7f5dbd4f396f6b90f36000b082f3 [file] [log] [blame]
Eugene Zelenko5adb96c2017-10-26 00:55:39 +00001//===- LoopVersioningLICM.cpp - LICM Loop Versioning ----------------------===//
Ashutosh Nemadf6763a2016-02-06 07:47:48 +00002//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// When alias analysis is uncertain about the aliasing between any two accesses,
11// it will return MayAlias. This uncertainty from alias analysis restricts LICM
12// from proceeding further. In cases where alias analysis is uncertain we might
13// use loop versioning as an alternative.
14//
15// Loop Versioning will create a version of the loop with aggressive aliasing
16// assumptions in addition to the original with conservative (default) aliasing
17// assumptions. The version of the loop making aggressive aliasing assumptions
18// will have all the memory accesses marked as no-alias. These two versions of
19// loop will be preceded by a memory runtime check. This runtime check consists
20// of bound checks for all unique memory accessed in loop, and it ensures the
21// lack of memory aliasing. The result of the runtime check determines which of
22// the loop versions is executed: If the runtime check detects any memory
23// aliasing, then the original loop is executed. Otherwise, the version with
24// aggressive aliasing assumptions is used.
25//
26// Following are the top level steps:
27//
28// a) Perform LoopVersioningLICM's feasibility check.
29// b) If loop is a candidate for versioning then create a memory bound check,
30// by considering all the memory accesses in loop body.
31// c) Clone original loop and set all memory accesses as no-alias in new loop.
32// d) Set original loop & versioned loop as a branch target of the runtime check
33// result.
34//
35// It transforms loop as shown below:
36//
37// +----------------+
38// |Runtime Memcheck|
39// +----------------+
40// |
41// +----------+----------------+----------+
42// | |
43// +---------+----------+ +-----------+----------+
44// |Orig Loop Preheader | |Cloned Loop Preheader |
45// +--------------------+ +----------------------+
46// | |
47// +--------------------+ +----------------------+
48// |Orig Loop Body | |Cloned Loop Body |
49// +--------------------+ +----------------------+
50// | |
51// +--------------------+ +----------------------+
Ashutosh Nema2260a3a2016-02-11 09:23:53 +000052// |Orig Loop Exit Block| |Cloned Loop Exit Block|
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000053// +--------------------+ +-----------+----------+
54// | |
55// +----------+--------------+-----------+
56// |
57// +-----+----+
58// |Join Block|
59// +----------+
60//
61//===----------------------------------------------------------------------===//
62
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000063#include "llvm/ADT/SmallVector.h"
64#include "llvm/ADT/StringRef.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000065#include "llvm/Analysis/AliasAnalysis.h"
66#include "llvm/Analysis/AliasSetTracker.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000067#include "llvm/Analysis/GlobalsModRef.h"
68#include "llvm/Analysis/LoopAccessAnalysis.h"
69#include "llvm/Analysis/LoopInfo.h"
70#include "llvm/Analysis/LoopPass.h"
Ashutosh Nema007b4252018-01-23 09:47:28 +000071#include "llvm/Analysis/OptimizationRemarkEmitter.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000072#include "llvm/Analysis/ScalarEvolution.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000073#include "llvm/IR/CallSite.h"
74#include "llvm/IR/Constants.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000075#include "llvm/IR/Dominators.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000076#include "llvm/IR/Instruction.h"
77#include "llvm/IR/Instructions.h"
78#include "llvm/IR/LLVMContext.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000079#include "llvm/IR/MDBuilder.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000080#include "llvm/IR/Metadata.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000081#include "llvm/IR/Type.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000082#include "llvm/IR/Value.h"
83#include "llvm/Pass.h"
84#include "llvm/Support/Casting.h"
85#include "llvm/Support/CommandLine.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000086#include "llvm/Support/Debug.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Transforms/Scalar.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000089#include "llvm/Transforms/Utils/LoopUtils.h"
90#include "llvm/Transforms/Utils/LoopVersioning.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000091#include <cassert>
92#include <memory>
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000093
94using namespace llvm;
95
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000096#define DEBUG_TYPE "loop-versioning-licm"
97
98static const char *LICMVersioningMetaData = "llvm.loop.licm_versioning.disable";
99
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000100/// Threshold minimum allowed percentage for possible
101/// invariant instructions in a loop.
102static cl::opt<float>
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000103 LVInvarThreshold("licm-versioning-invariant-threshold",
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000104 cl::desc("LoopVersioningLICM's minimum allowed percentage"
105 "of possible invariant instructions per loop"),
106 cl::init(25), cl::Hidden);
107
108/// Threshold for maximum allowed loop nest/depth
109static cl::opt<unsigned> LVLoopDepthThreshold(
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000110 "licm-versioning-max-depth-threshold",
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000111 cl::desc(
112 "LoopVersioningLICM's threshold for maximum allowed loop nest/depth"),
113 cl::init(2), cl::Hidden);
114
115/// \brief Create MDNode for input string.
116static MDNode *createStringMetadata(Loop *TheLoop, StringRef Name, unsigned V) {
117 LLVMContext &Context = TheLoop->getHeader()->getContext();
118 Metadata *MDs[] = {
119 MDString::get(Context, Name),
120 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(Context), V))};
121 return MDNode::get(Context, MDs);
122}
123
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000124/// \brief Set input string into loop metadata by keeping other values intact.
125void llvm::addStringMetadataToLoop(Loop *TheLoop, const char *MDString,
126 unsigned V) {
127 SmallVector<Metadata *, 4> MDs(1);
128 // If the loop already has metadata, retain it.
129 MDNode *LoopID = TheLoop->getLoopID();
130 if (LoopID) {
131 for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
132 MDNode *Node = cast<MDNode>(LoopID->getOperand(i));
133 MDs.push_back(Node);
134 }
135 }
136 // Add new metadata.
137 MDs.push_back(createStringMetadata(TheLoop, MDString, V));
138 // Replace current metadata node with new one.
139 LLVMContext &Context = TheLoop->getHeader()->getContext();
140 MDNode *NewLoopID = MDNode::get(Context, MDs);
141 // Set operand 0 to refer to the loop id itself.
142 NewLoopID->replaceOperandWith(0, NewLoopID);
143 TheLoop->setLoopID(NewLoopID);
144}
145
146namespace {
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000147
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000148struct LoopVersioningLICM : public LoopPass {
149 static char ID;
150
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000151 LoopVersioningLICM()
152 : LoopPass(ID), LoopDepthThreshold(LVLoopDepthThreshold),
153 InvariantThreshold(LVInvarThreshold) {
154 initializeLoopVersioningLICMPass(*PassRegistry::getPassRegistry());
155 }
156
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000157 bool runOnLoop(Loop *L, LPPassManager &LPM) override;
158
159 void getAnalysisUsage(AnalysisUsage &AU) const override {
160 AU.setPreservesCFG();
161 AU.addRequired<AAResultsWrapperPass>();
162 AU.addRequired<DominatorTreeWrapperPass>();
163 AU.addRequiredID(LCSSAID);
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000164 AU.addRequired<LoopAccessLegacyAnalysis>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000165 AU.addRequired<LoopInfoWrapperPass>();
166 AU.addRequiredID(LoopSimplifyID);
167 AU.addRequired<ScalarEvolutionWrapperPass>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000168 AU.addPreserved<AAResultsWrapperPass>();
169 AU.addPreserved<GlobalsAAWrapperPass>();
Ashutosh Nema007b4252018-01-23 09:47:28 +0000170 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000171 }
172
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000173 StringRef getPassName() const override { return "Loop Versioning for LICM"; }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000174
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000175 void reset() {
176 AA = nullptr;
177 SE = nullptr;
178 LAA = nullptr;
179 CurLoop = nullptr;
180 LoadAndStoreCounter = 0;
181 InvariantCounter = 0;
182 IsReadOnlyLoop = true;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000183 ORE = nullptr;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000184 CurAST.reset();
185 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000186
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000187 class AutoResetter {
188 public:
189 AutoResetter(LoopVersioningLICM &LVLICM) : LVLICM(LVLICM) {}
190 ~AutoResetter() { LVLICM.reset(); }
191
192 private:
193 LoopVersioningLICM &LVLICM;
194 };
195
196private:
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000197 // Current AliasAnalysis information
198 AliasAnalysis *AA = nullptr;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000199
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000200 // Current ScalarEvolution
201 ScalarEvolution *SE = nullptr;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000202
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000203 // Current LoopAccessAnalysis
204 LoopAccessLegacyAnalysis *LAA = nullptr;
205
206 // Current Loop's LoopAccessInfo
207 const LoopAccessInfo *LAI = nullptr;
208
209 // The current loop we are working on.
210 Loop *CurLoop = nullptr;
211
212 // AliasSet information for the current loop.
Ashutosh Nema007b4252018-01-23 09:47:28 +0000213 std::unique_ptr<AliasSetTracker> CurAST;
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000214
215 // Maximum loop nest threshold
216 unsigned LoopDepthThreshold;
217
218 // Minimum invariant threshold
219 float InvariantThreshold;
220
221 // Counter to track num of load & store
222 unsigned LoadAndStoreCounter = 0;
223
224 // Counter to track num of invariant
225 unsigned InvariantCounter = 0;
226
227 // Read only loop marker.
228 bool IsReadOnlyLoop = true;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000229
Ashutosh Nema007b4252018-01-23 09:47:28 +0000230 // OptimizationRemarkEmitter
231 OptimizationRemarkEmitter *ORE;
232
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000233 bool isLegalForVersioning();
234 bool legalLoopStructure();
235 bool legalLoopInstructions();
236 bool legalLoopMemoryAccesses();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000237 bool isLoopAlreadyVisited();
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000238 void setNoAliasToLoop(Loop *VerLoop);
239 bool instructionSafeForVersioning(Instruction *I);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000240};
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000241
242} // end anonymous namespace
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000243
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000244/// \brief Check loop structure and confirms it's good for LoopVersioningLICM.
245bool LoopVersioningLICM::legalLoopStructure() {
Florian Hahn2e032132016-12-19 17:13:37 +0000246 // Loop must be in loop simplify form.
247 if (!CurLoop->isLoopSimplifyForm()) {
248 DEBUG(
249 dbgs() << " loop is not in loop-simplify form.\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000250 return false;
251 }
252 // Loop should be innermost loop, if not return false.
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000253 if (!CurLoop->getSubLoops().empty()) {
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000254 DEBUG(dbgs() << " loop is not innermost\n");
255 return false;
256 }
257 // Loop should have a single backedge, if not return false.
258 if (CurLoop->getNumBackEdges() != 1) {
259 DEBUG(dbgs() << " loop has multiple backedges\n");
260 return false;
261 }
262 // Loop must have a single exiting block, if not return false.
263 if (!CurLoop->getExitingBlock()) {
264 DEBUG(dbgs() << " loop has multiple exiting block\n");
265 return false;
266 }
267 // We only handle bottom-tested loop, i.e. loop in which the condition is
268 // checked at the end of each iteration. With that we can assume that all
269 // instructions in the loop are executed the same number of times.
270 if (CurLoop->getExitingBlock() != CurLoop->getLoopLatch()) {
271 DEBUG(dbgs() << " loop is not bottom tested\n");
272 return false;
273 }
274 // Parallel loops must not have aliasing loop-invariant memory accesses.
275 // Hence we don't need to version anything in this case.
276 if (CurLoop->isAnnotatedParallel()) {
277 DEBUG(dbgs() << " Parallel loop is not worth versioning\n");
278 return false;
279 }
280 // Loop depth more then LoopDepthThreshold are not allowed
281 if (CurLoop->getLoopDepth() > LoopDepthThreshold) {
282 DEBUG(dbgs() << " loop depth is more then threshold\n");
283 return false;
284 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000285 // We need to be able to compute the loop trip count in order
286 // to generate the bound checks.
287 const SCEV *ExitCount = SE->getBackedgeTakenCount(CurLoop);
288 if (ExitCount == SE->getCouldNotCompute()) {
289 DEBUG(dbgs() << " loop does not has trip count\n");
290 return false;
291 }
292 return true;
293}
294
295/// \brief Check memory accesses in loop and confirms it's good for
296/// LoopVersioningLICM.
297bool LoopVersioningLICM::legalLoopMemoryAccesses() {
298 bool HasMayAlias = false;
299 bool TypeSafety = false;
300 bool HasMod = false;
301 // Memory check:
302 // Transform phase will generate a versioned loop and also a runtime check to
303 // ensure the pointers are independent and they don’t alias.
304 // In version variant of loop, alias meta data asserts that all access are
305 // mutually independent.
306 //
307 // Pointers aliasing in alias domain are avoided because with multiple
308 // aliasing domains we may not be able to hoist potential loop invariant
309 // access out of the loop.
310 //
311 // Iterate over alias tracker sets, and confirm AliasSets doesn't have any
312 // must alias set.
313 for (const auto &I : *CurAST) {
314 const AliasSet &AS = I;
315 // Skip Forward Alias Sets, as this should be ignored as part of
316 // the AliasSetTracker object.
317 if (AS.isForwardingAliasSet())
318 continue;
319 // With MustAlias its not worth adding runtime bound check.
320 if (AS.isMustAlias())
321 return false;
322 Value *SomePtr = AS.begin()->getValue();
323 bool TypeCheck = true;
324 // Check for Mod & MayAlias
325 HasMayAlias |= AS.isMayAlias();
326 HasMod |= AS.isMod();
327 for (const auto &A : AS) {
328 Value *Ptr = A.getValue();
329 // Alias tracker should have pointers of same data type.
330 TypeCheck = (TypeCheck && (SomePtr->getType() == Ptr->getType()));
331 }
332 // At least one alias tracker should have pointers of same data type.
333 TypeSafety |= TypeCheck;
334 }
335 // Ensure types should be of same type.
336 if (!TypeSafety) {
337 DEBUG(dbgs() << " Alias tracker type safety failed!\n");
338 return false;
339 }
340 // Ensure loop body shouldn't be read only.
341 if (!HasMod) {
342 DEBUG(dbgs() << " No memory modified in loop body\n");
343 return false;
344 }
345 // Make sure alias set has may alias case.
346 // If there no alias memory ambiguity, return false.
347 if (!HasMayAlias) {
348 DEBUG(dbgs() << " No ambiguity in memory access.\n");
349 return false;
350 }
351 return true;
352}
353
354/// \brief Check loop instructions safe for Loop versioning.
355/// It returns true if it's safe else returns false.
356/// Consider following:
357/// 1) Check all load store in loop body are non atomic & non volatile.
358/// 2) Check function call safety, by ensuring its not accessing memory.
359/// 3) Loop body shouldn't have any may throw instruction.
360bool LoopVersioningLICM::instructionSafeForVersioning(Instruction *I) {
361 assert(I != nullptr && "Null instruction found!");
362 // Check function call safety
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000363 if (isa<CallInst>(I) && !AA->doesNotAccessMemory(CallSite(I))) {
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000364 DEBUG(dbgs() << " Unsafe call site found.\n");
365 return false;
366 }
367 // Avoid loops with possiblity of throw
368 if (I->mayThrow()) {
369 DEBUG(dbgs() << " May throw instruction found in loop body\n");
370 return false;
371 }
372 // If current instruction is load instructions
373 // make sure it's a simple load (non atomic & non volatile)
374 if (I->mayReadFromMemory()) {
375 LoadInst *Ld = dyn_cast<LoadInst>(I);
376 if (!Ld || !Ld->isSimple()) {
377 DEBUG(dbgs() << " Found a non-simple load.\n");
378 return false;
379 }
380 LoadAndStoreCounter++;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000381 Value *Ptr = Ld->getPointerOperand();
382 // Check loop invariant.
383 if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
384 InvariantCounter++;
385 }
386 // If current instruction is store instruction
387 // make sure it's a simple store (non atomic & non volatile)
388 else if (I->mayWriteToMemory()) {
389 StoreInst *St = dyn_cast<StoreInst>(I);
390 if (!St || !St->isSimple()) {
391 DEBUG(dbgs() << " Found a non-simple store.\n");
392 return false;
393 }
394 LoadAndStoreCounter++;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000395 Value *Ptr = St->getPointerOperand();
396 // Check loop invariant.
397 if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
398 InvariantCounter++;
399
400 IsReadOnlyLoop = false;
401 }
402 return true;
403}
404
405/// \brief Check loop instructions and confirms it's good for
406/// LoopVersioningLICM.
407bool LoopVersioningLICM::legalLoopInstructions() {
408 // Resetting counters.
409 LoadAndStoreCounter = 0;
410 InvariantCounter = 0;
411 IsReadOnlyLoop = true;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000412 using namespace ore;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000413 // Iterate over loop blocks and instructions of each block and check
414 // instruction safety.
415 for (auto *Block : CurLoop->getBlocks())
416 for (auto &Inst : *Block) {
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000417 // If instruction is unsafe just return false.
Ashutosh Nema007b4252018-01-23 09:47:28 +0000418 if (!instructionSafeForVersioning(&Inst)) {
419 ORE->emit([&]() {
420 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopInst", &Inst)
421 << " Unsafe Loop Instruction";
422 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000423 return false;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000424 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000425 }
426 // Get LoopAccessInfo from current loop.
Adam Nemeta9f09c62016-06-17 22:35:41 +0000427 LAI = &LAA->getInfo(CurLoop);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000428 // Check LoopAccessInfo for need of runtime check.
429 if (LAI->getRuntimePointerChecking()->getChecks().empty()) {
430 DEBUG(dbgs() << " LAA: Runtime check not found !!\n");
431 return false;
432 }
433 // Number of runtime-checks should be less then RuntimeMemoryCheckThreshold
434 if (LAI->getNumRuntimePointerChecks() >
435 VectorizerParams::RuntimeMemoryCheckThreshold) {
436 DEBUG(dbgs() << " LAA: Runtime checks are more than threshold !!\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000437 ORE->emit([&]() {
438 return OptimizationRemarkMissed(DEBUG_TYPE, "RuntimeCheck",
439 CurLoop->getStartLoc(),
440 CurLoop->getHeader())
441 << "Number of runtime checks "
442 << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks())
443 << " exceeds threshold "
444 << NV("Threshold", VectorizerParams::RuntimeMemoryCheckThreshold);
445 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000446 return false;
447 }
448 // Loop should have at least one invariant load or store instruction.
449 if (!InvariantCounter) {
450 DEBUG(dbgs() << " Invariant not found !!\n");
451 return false;
452 }
453 // Read only loop not allowed.
454 if (IsReadOnlyLoop) {
455 DEBUG(dbgs() << " Found a read-only loop!\n");
456 return false;
457 }
458 // Profitablity check:
459 // Check invariant threshold, should be in limit.
460 if (InvariantCounter * 100 < InvariantThreshold * LoadAndStoreCounter) {
461 DEBUG(dbgs()
462 << " Invariant load & store are less then defined threshold\n");
463 DEBUG(dbgs() << " Invariant loads & stores: "
464 << ((InvariantCounter * 100) / LoadAndStoreCounter) << "%\n");
465 DEBUG(dbgs() << " Invariant loads & store threshold: "
466 << InvariantThreshold << "%\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000467 ORE->emit([&]() {
468 return OptimizationRemarkMissed(DEBUG_TYPE, "InvariantThreshold",
469 CurLoop->getStartLoc(),
470 CurLoop->getHeader())
471 << "Invariant load & store "
472 << NV("LoadAndStoreCounter",
473 ((InvariantCounter * 100) / LoadAndStoreCounter))
474 << " are less then defined threshold "
475 << NV("Threshold", InvariantThreshold);
476 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000477 return false;
478 }
479 return true;
480}
481
482/// \brief It checks loop is already visited or not.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000483/// check loop meta data, if loop revisited return true
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000484/// else false.
485bool LoopVersioningLICM::isLoopAlreadyVisited() {
486 // Check LoopVersioningLICM metadata into loop
Adam Nemetf7878262016-04-21 17:33:12 +0000487 if (findStringMetadataForLoop(CurLoop, LICMVersioningMetaData)) {
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000488 return true;
489 }
490 return false;
491}
492
493/// \brief Checks legality for LoopVersioningLICM by considering following:
494/// a) loop structure legality b) loop instruction legality
495/// c) loop memory access legality.
496/// Return true if legal else returns false.
497bool LoopVersioningLICM::isLegalForVersioning() {
Ashutosh Nema007b4252018-01-23 09:47:28 +0000498 using namespace ore;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000499 DEBUG(dbgs() << "Loop: " << *CurLoop);
500 // Make sure not re-visiting same loop again.
501 if (isLoopAlreadyVisited()) {
502 DEBUG(
503 dbgs() << " Revisiting loop in LoopVersioningLICM not allowed.\n\n");
504 return false;
505 }
506 // Check loop structure leagality.
507 if (!legalLoopStructure()) {
508 DEBUG(
509 dbgs() << " Loop structure not suitable for LoopVersioningLICM\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000510 ORE->emit([&]() {
511 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopStruct",
512 CurLoop->getStartLoc(),
513 CurLoop->getHeader())
514 << " Unsafe Loop structure";
515 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000516 return false;
517 }
518 // Check loop instruction leagality.
519 if (!legalLoopInstructions()) {
520 DEBUG(dbgs()
521 << " Loop instructions not suitable for LoopVersioningLICM\n\n");
522 return false;
523 }
524 // Check loop memory access leagality.
525 if (!legalLoopMemoryAccesses()) {
526 DEBUG(dbgs()
527 << " Loop memory access not suitable for LoopVersioningLICM\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000528 ORE->emit([&]() {
529 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopMemoryAccess",
530 CurLoop->getStartLoc(),
531 CurLoop->getHeader())
532 << " Unsafe Loop memory access";
533 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000534 return false;
535 }
536 // Loop versioning is feasible, return true.
537 DEBUG(dbgs() << " Loop Versioning found to be beneficial\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000538 ORE->emit([&]() {
539 return OptimizationRemark(DEBUG_TYPE, "IsLegalForVersioning",
540 CurLoop->getStartLoc(), CurLoop->getHeader())
541 << " Versioned loop for LICM."
542 << " Number of runtime checks we had to insert "
543 << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks());
544 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000545 return true;
546}
547
548/// \brief Update loop with aggressive aliasing assumptions.
549/// It marks no-alias to any pairs of memory operations by assuming
550/// loop should not have any must-alias memory accesses pairs.
551/// During LoopVersioningLICM legality we ignore loops having must
552/// aliasing memory accesses.
553void LoopVersioningLICM::setNoAliasToLoop(Loop *VerLoop) {
554 // Get latch terminator instruction.
555 Instruction *I = VerLoop->getLoopLatch()->getTerminator();
556 // Create alias scope domain.
557 MDBuilder MDB(I->getContext());
558 MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("LVDomain");
559 StringRef Name = "LVAliasScope";
560 SmallVector<Metadata *, 4> Scopes, NoAliases;
561 MDNode *NewScope = MDB.createAnonymousAliasScope(NewDomain, Name);
562 // Iterate over each instruction of loop.
563 // set no-alias for all load & store instructions.
564 for (auto *Block : CurLoop->getBlocks()) {
565 for (auto &Inst : *Block) {
566 // Only interested in instruction that may modify or read memory.
567 if (!Inst.mayReadFromMemory() && !Inst.mayWriteToMemory())
568 continue;
569 Scopes.push_back(NewScope);
570 NoAliases.push_back(NewScope);
571 // Set no-alias for current instruction.
572 Inst.setMetadata(
573 LLVMContext::MD_noalias,
574 MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_noalias),
575 MDNode::get(Inst.getContext(), NoAliases)));
576 // set alias-scope for current instruction.
577 Inst.setMetadata(
578 LLVMContext::MD_alias_scope,
579 MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_alias_scope),
580 MDNode::get(Inst.getContext(), Scopes)));
581 }
582 }
583}
584
585bool LoopVersioningLICM::runOnLoop(Loop *L, LPPassManager &LPM) {
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000586 // This will automatically release all resources hold by the current
587 // LoopVersioningLICM object.
588 AutoResetter Resetter(*this);
589
Andrew Kayloraa641a52016-04-22 22:06:11 +0000590 if (skipLoop(L))
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000591 return false;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000592 // Get Analysis information.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000593 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
594 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000595 LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
Ashutosh Nema007b4252018-01-23 09:47:28 +0000596 ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000597 LAI = nullptr;
598 // Set Current Loop
599 CurLoop = L;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000600 CurAST.reset(new AliasSetTracker(*AA));
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000601
602 // Loop over the body of this loop, construct AST.
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000603 LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000604 for (auto *Block : L->getBlocks()) {
605 if (LI->getLoopFor(Block) == L) // Ignore blocks in subloop.
606 CurAST->add(*Block); // Incorporate the specified basic block
607 }
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000608
609 bool Changed = false;
610
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000611 // Check feasiblity of LoopVersioningLICM.
612 // If versioning found to be feasible and beneficial then proceed
613 // else simply return, by cleaning up memory.
614 if (isLegalForVersioning()) {
615 // Do loop versioning.
616 // Create memcheck for memory accessed inside loop.
617 // Clone original loop, and set blocks properly.
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000618 DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000619 LoopVersioning LVer(*LAI, CurLoop, LI, DT, SE, true);
620 LVer.versionLoop();
621 // Set Loop Versioning metaData for original loop.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000622 addStringMetadataToLoop(LVer.getNonVersionedLoop(), LICMVersioningMetaData);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000623 // Set Loop Versioning metaData for version loop.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000624 addStringMetadataToLoop(LVer.getVersionedLoop(), LICMVersioningMetaData);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000625 // Set "llvm.mem.parallel_loop_access" metaData to versioned loop.
626 addStringMetadataToLoop(LVer.getVersionedLoop(),
627 "llvm.mem.parallel_loop_access");
628 // Update version loop with aggressive aliasing assumption.
629 setNoAliasToLoop(LVer.getVersionedLoop());
630 Changed = true;
631 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000632 return Changed;
633}
634
635char LoopVersioningLICM::ID = 0;
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000636
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000637INITIALIZE_PASS_BEGIN(LoopVersioningLICM, "loop-versioning-licm",
638 "Loop Versioning For LICM", false, false)
639INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
640INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
641INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
Easwaran Ramane12c4872016-06-09 19:44:46 +0000642INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass)
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000643INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000644INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
645INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
646INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
Ashutosh Nema007b4252018-01-23 09:47:28 +0000647INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000648INITIALIZE_PASS_END(LoopVersioningLICM, "loop-versioning-licm",
649 "Loop Versioning For LICM", false, false)
650
651Pass *llvm::createLoopVersioningLICMPass() { return new LoopVersioningLICM(); }