blob: ba75b8c705e1cb8ffece15e38391b58e3aaf0beb [file] [log] [blame]
Eugene Zelenko5adb96c2017-10-26 00:55:39 +00001//===- LoopVersioningLICM.cpp - LICM Loop Versioning ----------------------===//
Ashutosh Nemadf6763a2016-02-06 07:47:48 +00002//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// When alias analysis is uncertain about the aliasing between any two accesses,
11// it will return MayAlias. This uncertainty from alias analysis restricts LICM
12// from proceeding further. In cases where alias analysis is uncertain we might
13// use loop versioning as an alternative.
14//
15// Loop Versioning will create a version of the loop with aggressive aliasing
16// assumptions in addition to the original with conservative (default) aliasing
17// assumptions. The version of the loop making aggressive aliasing assumptions
18// will have all the memory accesses marked as no-alias. These two versions of
19// loop will be preceded by a memory runtime check. This runtime check consists
20// of bound checks for all unique memory accessed in loop, and it ensures the
21// lack of memory aliasing. The result of the runtime check determines which of
22// the loop versions is executed: If the runtime check detects any memory
23// aliasing, then the original loop is executed. Otherwise, the version with
24// aggressive aliasing assumptions is used.
25//
26// Following are the top level steps:
27//
28// a) Perform LoopVersioningLICM's feasibility check.
29// b) If loop is a candidate for versioning then create a memory bound check,
30// by considering all the memory accesses in loop body.
31// c) Clone original loop and set all memory accesses as no-alias in new loop.
32// d) Set original loop & versioned loop as a branch target of the runtime check
33// result.
34//
35// It transforms loop as shown below:
36//
37// +----------------+
38// |Runtime Memcheck|
39// +----------------+
40// |
41// +----------+----------------+----------+
42// | |
43// +---------+----------+ +-----------+----------+
44// |Orig Loop Preheader | |Cloned Loop Preheader |
45// +--------------------+ +----------------------+
46// | |
47// +--------------------+ +----------------------+
48// |Orig Loop Body | |Cloned Loop Body |
49// +--------------------+ +----------------------+
50// | |
51// +--------------------+ +----------------------+
Ashutosh Nema2260a3a2016-02-11 09:23:53 +000052// |Orig Loop Exit Block| |Cloned Loop Exit Block|
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000053// +--------------------+ +-----------+----------+
54// | |
55// +----------+--------------+-----------+
56// |
57// +-----+----+
58// |Join Block|
59// +----------+
60//
61//===----------------------------------------------------------------------===//
62
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000063#include "llvm/ADT/SmallVector.h"
64#include "llvm/ADT/StringRef.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000065#include "llvm/Analysis/AliasAnalysis.h"
66#include "llvm/Analysis/AliasSetTracker.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000067#include "llvm/Analysis/GlobalsModRef.h"
68#include "llvm/Analysis/LoopAccessAnalysis.h"
69#include "llvm/Analysis/LoopInfo.h"
70#include "llvm/Analysis/LoopPass.h"
Ashutosh Nema007b4252018-01-23 09:47:28 +000071#include "llvm/Analysis/OptimizationRemarkEmitter.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000072#include "llvm/Analysis/ScalarEvolution.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000073#include "llvm/IR/CallSite.h"
74#include "llvm/IR/Constants.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000075#include "llvm/IR/Dominators.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000076#include "llvm/IR/Instruction.h"
77#include "llvm/IR/Instructions.h"
78#include "llvm/IR/LLVMContext.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000079#include "llvm/IR/MDBuilder.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000080#include "llvm/IR/Metadata.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000081#include "llvm/IR/Type.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000082#include "llvm/IR/Value.h"
83#include "llvm/Pass.h"
84#include "llvm/Support/Casting.h"
85#include "llvm/Support/CommandLine.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000086#include "llvm/Support/Debug.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Transforms/Scalar.h"
David Blaikiea373d182018-03-28 17:44:36 +000089#include "llvm/Transforms/Utils.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000090#include "llvm/Transforms/Utils/LoopUtils.h"
91#include "llvm/Transforms/Utils/LoopVersioning.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000092#include <cassert>
93#include <memory>
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000094
95using namespace llvm;
96
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000097#define DEBUG_TYPE "loop-versioning-licm"
98
99static const char *LICMVersioningMetaData = "llvm.loop.licm_versioning.disable";
100
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000101/// Threshold minimum allowed percentage for possible
102/// invariant instructions in a loop.
103static cl::opt<float>
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000104 LVInvarThreshold("licm-versioning-invariant-threshold",
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000105 cl::desc("LoopVersioningLICM's minimum allowed percentage"
106 "of possible invariant instructions per loop"),
107 cl::init(25), cl::Hidden);
108
109/// Threshold for maximum allowed loop nest/depth
110static cl::opt<unsigned> LVLoopDepthThreshold(
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000111 "licm-versioning-max-depth-threshold",
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000112 cl::desc(
113 "LoopVersioningLICM's threshold for maximum allowed loop nest/depth"),
114 cl::init(2), cl::Hidden);
115
116/// \brief Create MDNode for input string.
117static MDNode *createStringMetadata(Loop *TheLoop, StringRef Name, unsigned V) {
118 LLVMContext &Context = TheLoop->getHeader()->getContext();
119 Metadata *MDs[] = {
120 MDString::get(Context, Name),
121 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(Context), V))};
122 return MDNode::get(Context, MDs);
123}
124
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000125/// \brief Set input string into loop metadata by keeping other values intact.
126void llvm::addStringMetadataToLoop(Loop *TheLoop, const char *MDString,
127 unsigned V) {
128 SmallVector<Metadata *, 4> MDs(1);
129 // If the loop already has metadata, retain it.
130 MDNode *LoopID = TheLoop->getLoopID();
131 if (LoopID) {
132 for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
133 MDNode *Node = cast<MDNode>(LoopID->getOperand(i));
134 MDs.push_back(Node);
135 }
136 }
137 // Add new metadata.
138 MDs.push_back(createStringMetadata(TheLoop, MDString, V));
139 // Replace current metadata node with new one.
140 LLVMContext &Context = TheLoop->getHeader()->getContext();
141 MDNode *NewLoopID = MDNode::get(Context, MDs);
142 // Set operand 0 to refer to the loop id itself.
143 NewLoopID->replaceOperandWith(0, NewLoopID);
144 TheLoop->setLoopID(NewLoopID);
145}
146
147namespace {
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000148
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000149struct LoopVersioningLICM : public LoopPass {
150 static char ID;
151
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000152 LoopVersioningLICM()
153 : LoopPass(ID), LoopDepthThreshold(LVLoopDepthThreshold),
154 InvariantThreshold(LVInvarThreshold) {
155 initializeLoopVersioningLICMPass(*PassRegistry::getPassRegistry());
156 }
157
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000158 bool runOnLoop(Loop *L, LPPassManager &LPM) override;
159
160 void getAnalysisUsage(AnalysisUsage &AU) const override {
161 AU.setPreservesCFG();
162 AU.addRequired<AAResultsWrapperPass>();
163 AU.addRequired<DominatorTreeWrapperPass>();
164 AU.addRequiredID(LCSSAID);
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000165 AU.addRequired<LoopAccessLegacyAnalysis>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000166 AU.addRequired<LoopInfoWrapperPass>();
167 AU.addRequiredID(LoopSimplifyID);
168 AU.addRequired<ScalarEvolutionWrapperPass>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000169 AU.addPreserved<AAResultsWrapperPass>();
170 AU.addPreserved<GlobalsAAWrapperPass>();
Ashutosh Nema007b4252018-01-23 09:47:28 +0000171 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000172 }
173
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000174 StringRef getPassName() const override { return "Loop Versioning for LICM"; }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000175
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000176 void reset() {
177 AA = nullptr;
178 SE = nullptr;
179 LAA = nullptr;
180 CurLoop = nullptr;
181 LoadAndStoreCounter = 0;
182 InvariantCounter = 0;
183 IsReadOnlyLoop = true;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000184 ORE = nullptr;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000185 CurAST.reset();
186 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000187
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000188 class AutoResetter {
189 public:
190 AutoResetter(LoopVersioningLICM &LVLICM) : LVLICM(LVLICM) {}
191 ~AutoResetter() { LVLICM.reset(); }
192
193 private:
194 LoopVersioningLICM &LVLICM;
195 };
196
197private:
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000198 // Current AliasAnalysis information
199 AliasAnalysis *AA = nullptr;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000200
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000201 // Current ScalarEvolution
202 ScalarEvolution *SE = nullptr;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000203
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000204 // Current LoopAccessAnalysis
205 LoopAccessLegacyAnalysis *LAA = nullptr;
206
207 // Current Loop's LoopAccessInfo
208 const LoopAccessInfo *LAI = nullptr;
209
210 // The current loop we are working on.
211 Loop *CurLoop = nullptr;
212
213 // AliasSet information for the current loop.
Ashutosh Nema007b4252018-01-23 09:47:28 +0000214 std::unique_ptr<AliasSetTracker> CurAST;
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000215
216 // Maximum loop nest threshold
217 unsigned LoopDepthThreshold;
218
219 // Minimum invariant threshold
220 float InvariantThreshold;
221
222 // Counter to track num of load & store
223 unsigned LoadAndStoreCounter = 0;
224
225 // Counter to track num of invariant
226 unsigned InvariantCounter = 0;
227
228 // Read only loop marker.
229 bool IsReadOnlyLoop = true;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000230
Ashutosh Nema007b4252018-01-23 09:47:28 +0000231 // OptimizationRemarkEmitter
232 OptimizationRemarkEmitter *ORE;
233
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000234 bool isLegalForVersioning();
235 bool legalLoopStructure();
236 bool legalLoopInstructions();
237 bool legalLoopMemoryAccesses();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000238 bool isLoopAlreadyVisited();
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000239 void setNoAliasToLoop(Loop *VerLoop);
240 bool instructionSafeForVersioning(Instruction *I);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000241};
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000242
243} // end anonymous namespace
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000244
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000245/// \brief Check loop structure and confirms it's good for LoopVersioningLICM.
246bool LoopVersioningLICM::legalLoopStructure() {
Florian Hahn2e032132016-12-19 17:13:37 +0000247 // Loop must be in loop simplify form.
248 if (!CurLoop->isLoopSimplifyForm()) {
249 DEBUG(
250 dbgs() << " loop is not in loop-simplify form.\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000251 return false;
252 }
253 // Loop should be innermost loop, if not return false.
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000254 if (!CurLoop->getSubLoops().empty()) {
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000255 DEBUG(dbgs() << " loop is not innermost\n");
256 return false;
257 }
258 // Loop should have a single backedge, if not return false.
259 if (CurLoop->getNumBackEdges() != 1) {
260 DEBUG(dbgs() << " loop has multiple backedges\n");
261 return false;
262 }
263 // Loop must have a single exiting block, if not return false.
264 if (!CurLoop->getExitingBlock()) {
265 DEBUG(dbgs() << " loop has multiple exiting block\n");
266 return false;
267 }
268 // We only handle bottom-tested loop, i.e. loop in which the condition is
269 // checked at the end of each iteration. With that we can assume that all
270 // instructions in the loop are executed the same number of times.
271 if (CurLoop->getExitingBlock() != CurLoop->getLoopLatch()) {
272 DEBUG(dbgs() << " loop is not bottom tested\n");
273 return false;
274 }
275 // Parallel loops must not have aliasing loop-invariant memory accesses.
276 // Hence we don't need to version anything in this case.
277 if (CurLoop->isAnnotatedParallel()) {
278 DEBUG(dbgs() << " Parallel loop is not worth versioning\n");
279 return false;
280 }
281 // Loop depth more then LoopDepthThreshold are not allowed
282 if (CurLoop->getLoopDepth() > LoopDepthThreshold) {
283 DEBUG(dbgs() << " loop depth is more then threshold\n");
284 return false;
285 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000286 // We need to be able to compute the loop trip count in order
287 // to generate the bound checks.
288 const SCEV *ExitCount = SE->getBackedgeTakenCount(CurLoop);
289 if (ExitCount == SE->getCouldNotCompute()) {
290 DEBUG(dbgs() << " loop does not has trip count\n");
291 return false;
292 }
293 return true;
294}
295
296/// \brief Check memory accesses in loop and confirms it's good for
297/// LoopVersioningLICM.
298bool LoopVersioningLICM::legalLoopMemoryAccesses() {
299 bool HasMayAlias = false;
300 bool TypeSafety = false;
301 bool HasMod = false;
302 // Memory check:
303 // Transform phase will generate a versioned loop and also a runtime check to
304 // ensure the pointers are independent and they don’t alias.
305 // In version variant of loop, alias meta data asserts that all access are
306 // mutually independent.
307 //
308 // Pointers aliasing in alias domain are avoided because with multiple
309 // aliasing domains we may not be able to hoist potential loop invariant
310 // access out of the loop.
311 //
312 // Iterate over alias tracker sets, and confirm AliasSets doesn't have any
313 // must alias set.
314 for (const auto &I : *CurAST) {
315 const AliasSet &AS = I;
316 // Skip Forward Alias Sets, as this should be ignored as part of
317 // the AliasSetTracker object.
318 if (AS.isForwardingAliasSet())
319 continue;
320 // With MustAlias its not worth adding runtime bound check.
321 if (AS.isMustAlias())
322 return false;
323 Value *SomePtr = AS.begin()->getValue();
324 bool TypeCheck = true;
325 // Check for Mod & MayAlias
326 HasMayAlias |= AS.isMayAlias();
327 HasMod |= AS.isMod();
328 for (const auto &A : AS) {
329 Value *Ptr = A.getValue();
330 // Alias tracker should have pointers of same data type.
331 TypeCheck = (TypeCheck && (SomePtr->getType() == Ptr->getType()));
332 }
333 // At least one alias tracker should have pointers of same data type.
334 TypeSafety |= TypeCheck;
335 }
336 // Ensure types should be of same type.
337 if (!TypeSafety) {
338 DEBUG(dbgs() << " Alias tracker type safety failed!\n");
339 return false;
340 }
341 // Ensure loop body shouldn't be read only.
342 if (!HasMod) {
343 DEBUG(dbgs() << " No memory modified in loop body\n");
344 return false;
345 }
346 // Make sure alias set has may alias case.
347 // If there no alias memory ambiguity, return false.
348 if (!HasMayAlias) {
349 DEBUG(dbgs() << " No ambiguity in memory access.\n");
350 return false;
351 }
352 return true;
353}
354
355/// \brief Check loop instructions safe for Loop versioning.
356/// It returns true if it's safe else returns false.
357/// Consider following:
358/// 1) Check all load store in loop body are non atomic & non volatile.
359/// 2) Check function call safety, by ensuring its not accessing memory.
360/// 3) Loop body shouldn't have any may throw instruction.
361bool LoopVersioningLICM::instructionSafeForVersioning(Instruction *I) {
362 assert(I != nullptr && "Null instruction found!");
363 // Check function call safety
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000364 if (isa<CallInst>(I) && !AA->doesNotAccessMemory(CallSite(I))) {
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000365 DEBUG(dbgs() << " Unsafe call site found.\n");
366 return false;
367 }
368 // Avoid loops with possiblity of throw
369 if (I->mayThrow()) {
370 DEBUG(dbgs() << " May throw instruction found in loop body\n");
371 return false;
372 }
373 // If current instruction is load instructions
374 // make sure it's a simple load (non atomic & non volatile)
375 if (I->mayReadFromMemory()) {
376 LoadInst *Ld = dyn_cast<LoadInst>(I);
377 if (!Ld || !Ld->isSimple()) {
378 DEBUG(dbgs() << " Found a non-simple load.\n");
379 return false;
380 }
381 LoadAndStoreCounter++;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000382 Value *Ptr = Ld->getPointerOperand();
383 // Check loop invariant.
384 if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
385 InvariantCounter++;
386 }
387 // If current instruction is store instruction
388 // make sure it's a simple store (non atomic & non volatile)
389 else if (I->mayWriteToMemory()) {
390 StoreInst *St = dyn_cast<StoreInst>(I);
391 if (!St || !St->isSimple()) {
392 DEBUG(dbgs() << " Found a non-simple store.\n");
393 return false;
394 }
395 LoadAndStoreCounter++;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000396 Value *Ptr = St->getPointerOperand();
397 // Check loop invariant.
398 if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
399 InvariantCounter++;
400
401 IsReadOnlyLoop = false;
402 }
403 return true;
404}
405
406/// \brief Check loop instructions and confirms it's good for
407/// LoopVersioningLICM.
408bool LoopVersioningLICM::legalLoopInstructions() {
409 // Resetting counters.
410 LoadAndStoreCounter = 0;
411 InvariantCounter = 0;
412 IsReadOnlyLoop = true;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000413 using namespace ore;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000414 // Iterate over loop blocks and instructions of each block and check
415 // instruction safety.
416 for (auto *Block : CurLoop->getBlocks())
417 for (auto &Inst : *Block) {
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000418 // If instruction is unsafe just return false.
Ashutosh Nema007b4252018-01-23 09:47:28 +0000419 if (!instructionSafeForVersioning(&Inst)) {
420 ORE->emit([&]() {
421 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopInst", &Inst)
422 << " Unsafe Loop Instruction";
423 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000424 return false;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000425 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000426 }
427 // Get LoopAccessInfo from current loop.
Adam Nemeta9f09c62016-06-17 22:35:41 +0000428 LAI = &LAA->getInfo(CurLoop);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000429 // Check LoopAccessInfo for need of runtime check.
430 if (LAI->getRuntimePointerChecking()->getChecks().empty()) {
431 DEBUG(dbgs() << " LAA: Runtime check not found !!\n");
432 return false;
433 }
434 // Number of runtime-checks should be less then RuntimeMemoryCheckThreshold
435 if (LAI->getNumRuntimePointerChecks() >
436 VectorizerParams::RuntimeMemoryCheckThreshold) {
437 DEBUG(dbgs() << " LAA: Runtime checks are more than threshold !!\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000438 ORE->emit([&]() {
439 return OptimizationRemarkMissed(DEBUG_TYPE, "RuntimeCheck",
440 CurLoop->getStartLoc(),
441 CurLoop->getHeader())
442 << "Number of runtime checks "
443 << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks())
444 << " exceeds threshold "
445 << NV("Threshold", VectorizerParams::RuntimeMemoryCheckThreshold);
446 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000447 return false;
448 }
449 // Loop should have at least one invariant load or store instruction.
450 if (!InvariantCounter) {
451 DEBUG(dbgs() << " Invariant not found !!\n");
452 return false;
453 }
454 // Read only loop not allowed.
455 if (IsReadOnlyLoop) {
456 DEBUG(dbgs() << " Found a read-only loop!\n");
457 return false;
458 }
459 // Profitablity check:
460 // Check invariant threshold, should be in limit.
461 if (InvariantCounter * 100 < InvariantThreshold * LoadAndStoreCounter) {
462 DEBUG(dbgs()
463 << " Invariant load & store are less then defined threshold\n");
464 DEBUG(dbgs() << " Invariant loads & stores: "
465 << ((InvariantCounter * 100) / LoadAndStoreCounter) << "%\n");
466 DEBUG(dbgs() << " Invariant loads & store threshold: "
467 << InvariantThreshold << "%\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000468 ORE->emit([&]() {
469 return OptimizationRemarkMissed(DEBUG_TYPE, "InvariantThreshold",
470 CurLoop->getStartLoc(),
471 CurLoop->getHeader())
472 << "Invariant load & store "
473 << NV("LoadAndStoreCounter",
474 ((InvariantCounter * 100) / LoadAndStoreCounter))
475 << " are less then defined threshold "
476 << NV("Threshold", InvariantThreshold);
477 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000478 return false;
479 }
480 return true;
481}
482
483/// \brief It checks loop is already visited or not.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000484/// check loop meta data, if loop revisited return true
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000485/// else false.
486bool LoopVersioningLICM::isLoopAlreadyVisited() {
487 // Check LoopVersioningLICM metadata into loop
Adam Nemetf7878262016-04-21 17:33:12 +0000488 if (findStringMetadataForLoop(CurLoop, LICMVersioningMetaData)) {
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000489 return true;
490 }
491 return false;
492}
493
494/// \brief Checks legality for LoopVersioningLICM by considering following:
495/// a) loop structure legality b) loop instruction legality
496/// c) loop memory access legality.
497/// Return true if legal else returns false.
498bool LoopVersioningLICM::isLegalForVersioning() {
Ashutosh Nema007b4252018-01-23 09:47:28 +0000499 using namespace ore;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000500 DEBUG(dbgs() << "Loop: " << *CurLoop);
501 // Make sure not re-visiting same loop again.
502 if (isLoopAlreadyVisited()) {
503 DEBUG(
504 dbgs() << " Revisiting loop in LoopVersioningLICM not allowed.\n\n");
505 return false;
506 }
507 // Check loop structure leagality.
508 if (!legalLoopStructure()) {
509 DEBUG(
510 dbgs() << " Loop structure not suitable for LoopVersioningLICM\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000511 ORE->emit([&]() {
512 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopStruct",
513 CurLoop->getStartLoc(),
514 CurLoop->getHeader())
515 << " Unsafe Loop structure";
516 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000517 return false;
518 }
519 // Check loop instruction leagality.
520 if (!legalLoopInstructions()) {
521 DEBUG(dbgs()
522 << " Loop instructions not suitable for LoopVersioningLICM\n\n");
523 return false;
524 }
525 // Check loop memory access leagality.
526 if (!legalLoopMemoryAccesses()) {
527 DEBUG(dbgs()
528 << " Loop memory access not suitable for LoopVersioningLICM\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000529 ORE->emit([&]() {
530 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopMemoryAccess",
531 CurLoop->getStartLoc(),
532 CurLoop->getHeader())
533 << " Unsafe Loop memory access";
534 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000535 return false;
536 }
537 // Loop versioning is feasible, return true.
538 DEBUG(dbgs() << " Loop Versioning found to be beneficial\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000539 ORE->emit([&]() {
540 return OptimizationRemark(DEBUG_TYPE, "IsLegalForVersioning",
541 CurLoop->getStartLoc(), CurLoop->getHeader())
542 << " Versioned loop for LICM."
543 << " Number of runtime checks we had to insert "
544 << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks());
545 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000546 return true;
547}
548
549/// \brief Update loop with aggressive aliasing assumptions.
550/// It marks no-alias to any pairs of memory operations by assuming
551/// loop should not have any must-alias memory accesses pairs.
552/// During LoopVersioningLICM legality we ignore loops having must
553/// aliasing memory accesses.
554void LoopVersioningLICM::setNoAliasToLoop(Loop *VerLoop) {
555 // Get latch terminator instruction.
556 Instruction *I = VerLoop->getLoopLatch()->getTerminator();
557 // Create alias scope domain.
558 MDBuilder MDB(I->getContext());
559 MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("LVDomain");
560 StringRef Name = "LVAliasScope";
561 SmallVector<Metadata *, 4> Scopes, NoAliases;
562 MDNode *NewScope = MDB.createAnonymousAliasScope(NewDomain, Name);
563 // Iterate over each instruction of loop.
564 // set no-alias for all load & store instructions.
565 for (auto *Block : CurLoop->getBlocks()) {
566 for (auto &Inst : *Block) {
567 // Only interested in instruction that may modify or read memory.
568 if (!Inst.mayReadFromMemory() && !Inst.mayWriteToMemory())
569 continue;
570 Scopes.push_back(NewScope);
571 NoAliases.push_back(NewScope);
572 // Set no-alias for current instruction.
573 Inst.setMetadata(
574 LLVMContext::MD_noalias,
575 MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_noalias),
576 MDNode::get(Inst.getContext(), NoAliases)));
577 // set alias-scope for current instruction.
578 Inst.setMetadata(
579 LLVMContext::MD_alias_scope,
580 MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_alias_scope),
581 MDNode::get(Inst.getContext(), Scopes)));
582 }
583 }
584}
585
586bool LoopVersioningLICM::runOnLoop(Loop *L, LPPassManager &LPM) {
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000587 // This will automatically release all resources hold by the current
588 // LoopVersioningLICM object.
589 AutoResetter Resetter(*this);
590
Andrew Kayloraa641a52016-04-22 22:06:11 +0000591 if (skipLoop(L))
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000592 return false;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000593 // Get Analysis information.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000594 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
595 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000596 LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
Ashutosh Nema007b4252018-01-23 09:47:28 +0000597 ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000598 LAI = nullptr;
599 // Set Current Loop
600 CurLoop = L;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000601 CurAST.reset(new AliasSetTracker(*AA));
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000602
603 // Loop over the body of this loop, construct AST.
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000604 LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000605 for (auto *Block : L->getBlocks()) {
606 if (LI->getLoopFor(Block) == L) // Ignore blocks in subloop.
607 CurAST->add(*Block); // Incorporate the specified basic block
608 }
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000609
610 bool Changed = false;
611
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000612 // Check feasiblity of LoopVersioningLICM.
613 // If versioning found to be feasible and beneficial then proceed
614 // else simply return, by cleaning up memory.
615 if (isLegalForVersioning()) {
616 // Do loop versioning.
617 // Create memcheck for memory accessed inside loop.
618 // Clone original loop, and set blocks properly.
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000619 DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000620 LoopVersioning LVer(*LAI, CurLoop, LI, DT, SE, true);
621 LVer.versionLoop();
622 // Set Loop Versioning metaData for original loop.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000623 addStringMetadataToLoop(LVer.getNonVersionedLoop(), LICMVersioningMetaData);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000624 // Set Loop Versioning metaData for version loop.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000625 addStringMetadataToLoop(LVer.getVersionedLoop(), LICMVersioningMetaData);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000626 // Set "llvm.mem.parallel_loop_access" metaData to versioned loop.
627 addStringMetadataToLoop(LVer.getVersionedLoop(),
628 "llvm.mem.parallel_loop_access");
629 // Update version loop with aggressive aliasing assumption.
630 setNoAliasToLoop(LVer.getVersionedLoop());
631 Changed = true;
632 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000633 return Changed;
634}
635
636char LoopVersioningLICM::ID = 0;
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000637
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000638INITIALIZE_PASS_BEGIN(LoopVersioningLICM, "loop-versioning-licm",
639 "Loop Versioning For LICM", false, false)
640INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
641INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
642INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
Easwaran Ramane12c4872016-06-09 19:44:46 +0000643INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass)
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000644INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000645INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
646INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
647INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
Ashutosh Nema007b4252018-01-23 09:47:28 +0000648INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000649INITIALIZE_PASS_END(LoopVersioningLICM, "loop-versioning-licm",
650 "Loop Versioning For LICM", false, false)
651
652Pass *llvm::createLoopVersioningLICMPass() { return new LoopVersioningLICM(); }