blob: 2ccb7cae3079ee746c1efac4bcf6114da0d106c0 [file] [log] [blame]
Eugene Zelenko5adb96c2017-10-26 00:55:39 +00001//===- LoopVersioningLICM.cpp - LICM Loop Versioning ----------------------===//
Ashutosh Nemadf6763a2016-02-06 07:47:48 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Ashutosh Nemadf6763a2016-02-06 07:47:48 +00006//
7//===----------------------------------------------------------------------===//
8//
9// When alias analysis is uncertain about the aliasing between any two accesses,
10// it will return MayAlias. This uncertainty from alias analysis restricts LICM
11// from proceeding further. In cases where alias analysis is uncertain we might
12// use loop versioning as an alternative.
13//
14// Loop Versioning will create a version of the loop with aggressive aliasing
15// assumptions in addition to the original with conservative (default) aliasing
16// assumptions. The version of the loop making aggressive aliasing assumptions
17// will have all the memory accesses marked as no-alias. These two versions of
18// loop will be preceded by a memory runtime check. This runtime check consists
19// of bound checks for all unique memory accessed in loop, and it ensures the
20// lack of memory aliasing. The result of the runtime check determines which of
21// the loop versions is executed: If the runtime check detects any memory
22// aliasing, then the original loop is executed. Otherwise, the version with
23// aggressive aliasing assumptions is used.
24//
25// Following are the top level steps:
26//
27// a) Perform LoopVersioningLICM's feasibility check.
28// b) If loop is a candidate for versioning then create a memory bound check,
29// by considering all the memory accesses in loop body.
30// c) Clone original loop and set all memory accesses as no-alias in new loop.
31// d) Set original loop & versioned loop as a branch target of the runtime check
32// result.
33//
34// It transforms loop as shown below:
35//
36// +----------------+
37// |Runtime Memcheck|
38// +----------------+
39// |
40// +----------+----------------+----------+
41// | |
42// +---------+----------+ +-----------+----------+
43// |Orig Loop Preheader | |Cloned Loop Preheader |
44// +--------------------+ +----------------------+
45// | |
46// +--------------------+ +----------------------+
47// |Orig Loop Body | |Cloned Loop Body |
48// +--------------------+ +----------------------+
49// | |
50// +--------------------+ +----------------------+
Ashutosh Nema2260a3a2016-02-11 09:23:53 +000051// |Orig Loop Exit Block| |Cloned Loop Exit Block|
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000052// +--------------------+ +-----------+----------+
53// | |
54// +----------+--------------+-----------+
55// |
56// +-----+----+
57// |Join Block|
58// +----------+
59//
60//===----------------------------------------------------------------------===//
61
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000062#include "llvm/ADT/SmallVector.h"
63#include "llvm/ADT/StringRef.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000064#include "llvm/Analysis/AliasAnalysis.h"
65#include "llvm/Analysis/AliasSetTracker.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000066#include "llvm/Analysis/GlobalsModRef.h"
67#include "llvm/Analysis/LoopAccessAnalysis.h"
68#include "llvm/Analysis/LoopInfo.h"
69#include "llvm/Analysis/LoopPass.h"
Ashutosh Nema007b4252018-01-23 09:47:28 +000070#include "llvm/Analysis/OptimizationRemarkEmitter.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000071#include "llvm/Analysis/ScalarEvolution.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000072#include "llvm/IR/CallSite.h"
73#include "llvm/IR/Constants.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000074#include "llvm/IR/Dominators.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000075#include "llvm/IR/Instruction.h"
76#include "llvm/IR/Instructions.h"
77#include "llvm/IR/LLVMContext.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000078#include "llvm/IR/MDBuilder.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000079#include "llvm/IR/Metadata.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000080#include "llvm/IR/Type.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000081#include "llvm/IR/Value.h"
82#include "llvm/Pass.h"
83#include "llvm/Support/Casting.h"
84#include "llvm/Support/CommandLine.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000085#include "llvm/Support/Debug.h"
86#include "llvm/Support/raw_ostream.h"
87#include "llvm/Transforms/Scalar.h"
David Blaikiea373d182018-03-28 17:44:36 +000088#include "llvm/Transforms/Utils.h"
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000089#include "llvm/Transforms/Utils/LoopUtils.h"
90#include "llvm/Transforms/Utils/LoopVersioning.h"
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000091#include <cassert>
92#include <memory>
Ashutosh Nemadf6763a2016-02-06 07:47:48 +000093
94using namespace llvm;
95
Eugene Zelenko5adb96c2017-10-26 00:55:39 +000096#define DEBUG_TYPE "loop-versioning-licm"
97
98static const char *LICMVersioningMetaData = "llvm.loop.licm_versioning.disable";
99
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000100/// Threshold minimum allowed percentage for possible
101/// invariant instructions in a loop.
102static cl::opt<float>
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000103 LVInvarThreshold("licm-versioning-invariant-threshold",
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000104 cl::desc("LoopVersioningLICM's minimum allowed percentage"
105 "of possible invariant instructions per loop"),
106 cl::init(25), cl::Hidden);
107
108/// Threshold for maximum allowed loop nest/depth
109static cl::opt<unsigned> LVLoopDepthThreshold(
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000110 "licm-versioning-max-depth-threshold",
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000111 cl::desc(
112 "LoopVersioningLICM's threshold for maximum allowed loop nest/depth"),
113 cl::init(2), cl::Hidden);
114
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000115namespace {
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000116
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000117struct LoopVersioningLICM : public LoopPass {
118 static char ID;
119
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000120 LoopVersioningLICM()
121 : LoopPass(ID), LoopDepthThreshold(LVLoopDepthThreshold),
122 InvariantThreshold(LVInvarThreshold) {
123 initializeLoopVersioningLICMPass(*PassRegistry::getPassRegistry());
124 }
125
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000126 bool runOnLoop(Loop *L, LPPassManager &LPM) override;
127
128 void getAnalysisUsage(AnalysisUsage &AU) const override {
129 AU.setPreservesCFG();
130 AU.addRequired<AAResultsWrapperPass>();
131 AU.addRequired<DominatorTreeWrapperPass>();
132 AU.addRequiredID(LCSSAID);
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000133 AU.addRequired<LoopAccessLegacyAnalysis>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000134 AU.addRequired<LoopInfoWrapperPass>();
135 AU.addRequiredID(LoopSimplifyID);
136 AU.addRequired<ScalarEvolutionWrapperPass>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000137 AU.addPreserved<AAResultsWrapperPass>();
138 AU.addPreserved<GlobalsAAWrapperPass>();
Ashutosh Nema007b4252018-01-23 09:47:28 +0000139 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000140 }
141
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000142 StringRef getPassName() const override { return "Loop Versioning for LICM"; }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000143
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000144 void reset() {
145 AA = nullptr;
146 SE = nullptr;
147 LAA = nullptr;
148 CurLoop = nullptr;
149 LoadAndStoreCounter = 0;
150 InvariantCounter = 0;
151 IsReadOnlyLoop = true;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000152 ORE = nullptr;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000153 CurAST.reset();
154 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000155
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000156 class AutoResetter {
157 public:
158 AutoResetter(LoopVersioningLICM &LVLICM) : LVLICM(LVLICM) {}
159 ~AutoResetter() { LVLICM.reset(); }
160
161 private:
162 LoopVersioningLICM &LVLICM;
163 };
164
165private:
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000166 // Current AliasAnalysis information
167 AliasAnalysis *AA = nullptr;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000168
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000169 // Current ScalarEvolution
170 ScalarEvolution *SE = nullptr;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000171
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000172 // Current LoopAccessAnalysis
173 LoopAccessLegacyAnalysis *LAA = nullptr;
174
175 // Current Loop's LoopAccessInfo
176 const LoopAccessInfo *LAI = nullptr;
177
178 // The current loop we are working on.
179 Loop *CurLoop = nullptr;
180
181 // AliasSet information for the current loop.
Ashutosh Nema007b4252018-01-23 09:47:28 +0000182 std::unique_ptr<AliasSetTracker> CurAST;
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000183
184 // Maximum loop nest threshold
185 unsigned LoopDepthThreshold;
186
187 // Minimum invariant threshold
188 float InvariantThreshold;
189
190 // Counter to track num of load & store
191 unsigned LoadAndStoreCounter = 0;
192
193 // Counter to track num of invariant
194 unsigned InvariantCounter = 0;
195
196 // Read only loop marker.
197 bool IsReadOnlyLoop = true;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000198
Ashutosh Nema007b4252018-01-23 09:47:28 +0000199 // OptimizationRemarkEmitter
200 OptimizationRemarkEmitter *ORE;
201
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000202 bool isLegalForVersioning();
203 bool legalLoopStructure();
204 bool legalLoopInstructions();
205 bool legalLoopMemoryAccesses();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000206 bool isLoopAlreadyVisited();
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000207 void setNoAliasToLoop(Loop *VerLoop);
208 bool instructionSafeForVersioning(Instruction *I);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000209};
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000210
211} // end anonymous namespace
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000212
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000213/// Check loop structure and confirms it's good for LoopVersioningLICM.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000214bool LoopVersioningLICM::legalLoopStructure() {
Florian Hahn2e032132016-12-19 17:13:37 +0000215 // Loop must be in loop simplify form.
216 if (!CurLoop->isLoopSimplifyForm()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000217 LLVM_DEBUG(dbgs() << " loop is not in loop-simplify form.\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000218 return false;
219 }
220 // Loop should be innermost loop, if not return false.
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000221 if (!CurLoop->getSubLoops().empty()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000222 LLVM_DEBUG(dbgs() << " loop is not innermost\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000223 return false;
224 }
225 // Loop should have a single backedge, if not return false.
226 if (CurLoop->getNumBackEdges() != 1) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000227 LLVM_DEBUG(dbgs() << " loop has multiple backedges\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000228 return false;
229 }
230 // Loop must have a single exiting block, if not return false.
231 if (!CurLoop->getExitingBlock()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000232 LLVM_DEBUG(dbgs() << " loop has multiple exiting block\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000233 return false;
234 }
235 // We only handle bottom-tested loop, i.e. loop in which the condition is
236 // checked at the end of each iteration. With that we can assume that all
237 // instructions in the loop are executed the same number of times.
238 if (CurLoop->getExitingBlock() != CurLoop->getLoopLatch()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000239 LLVM_DEBUG(dbgs() << " loop is not bottom tested\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000240 return false;
241 }
242 // Parallel loops must not have aliasing loop-invariant memory accesses.
243 // Hence we don't need to version anything in this case.
244 if (CurLoop->isAnnotatedParallel()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000245 LLVM_DEBUG(dbgs() << " Parallel loop is not worth versioning\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000246 return false;
247 }
248 // Loop depth more then LoopDepthThreshold are not allowed
249 if (CurLoop->getLoopDepth() > LoopDepthThreshold) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000250 LLVM_DEBUG(dbgs() << " loop depth is more then threshold\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000251 return false;
252 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000253 // We need to be able to compute the loop trip count in order
254 // to generate the bound checks.
255 const SCEV *ExitCount = SE->getBackedgeTakenCount(CurLoop);
256 if (ExitCount == SE->getCouldNotCompute()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000257 LLVM_DEBUG(dbgs() << " loop does not has trip count\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000258 return false;
259 }
260 return true;
261}
262
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000263/// Check memory accesses in loop and confirms it's good for
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000264/// LoopVersioningLICM.
265bool LoopVersioningLICM::legalLoopMemoryAccesses() {
266 bool HasMayAlias = false;
267 bool TypeSafety = false;
268 bool HasMod = false;
269 // Memory check:
270 // Transform phase will generate a versioned loop and also a runtime check to
271 // ensure the pointers are independent and they don’t alias.
272 // In version variant of loop, alias meta data asserts that all access are
273 // mutually independent.
274 //
275 // Pointers aliasing in alias domain are avoided because with multiple
276 // aliasing domains we may not be able to hoist potential loop invariant
277 // access out of the loop.
278 //
279 // Iterate over alias tracker sets, and confirm AliasSets doesn't have any
280 // must alias set.
281 for (const auto &I : *CurAST) {
282 const AliasSet &AS = I;
283 // Skip Forward Alias Sets, as this should be ignored as part of
284 // the AliasSetTracker object.
285 if (AS.isForwardingAliasSet())
286 continue;
287 // With MustAlias its not worth adding runtime bound check.
288 if (AS.isMustAlias())
289 return false;
290 Value *SomePtr = AS.begin()->getValue();
291 bool TypeCheck = true;
292 // Check for Mod & MayAlias
293 HasMayAlias |= AS.isMayAlias();
294 HasMod |= AS.isMod();
295 for (const auto &A : AS) {
296 Value *Ptr = A.getValue();
297 // Alias tracker should have pointers of same data type.
298 TypeCheck = (TypeCheck && (SomePtr->getType() == Ptr->getType()));
299 }
300 // At least one alias tracker should have pointers of same data type.
301 TypeSafety |= TypeCheck;
302 }
303 // Ensure types should be of same type.
304 if (!TypeSafety) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000305 LLVM_DEBUG(dbgs() << " Alias tracker type safety failed!\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000306 return false;
307 }
308 // Ensure loop body shouldn't be read only.
309 if (!HasMod) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000310 LLVM_DEBUG(dbgs() << " No memory modified in loop body\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000311 return false;
312 }
313 // Make sure alias set has may alias case.
314 // If there no alias memory ambiguity, return false.
315 if (!HasMayAlias) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000316 LLVM_DEBUG(dbgs() << " No ambiguity in memory access.\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000317 return false;
318 }
319 return true;
320}
321
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000322/// Check loop instructions safe for Loop versioning.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000323/// It returns true if it's safe else returns false.
324/// Consider following:
325/// 1) Check all load store in loop body are non atomic & non volatile.
326/// 2) Check function call safety, by ensuring its not accessing memory.
327/// 3) Loop body shouldn't have any may throw instruction.
Matt Arsenault79b3ea72019-05-29 20:47:59 +0000328/// 4) Loop body shouldn't have any convergent or noduplicate instructions.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000329bool LoopVersioningLICM::instructionSafeForVersioning(Instruction *I) {
330 assert(I != nullptr && "Null instruction found!");
331 // Check function call safety
Matt Arsenault79b3ea72019-05-29 20:47:59 +0000332 if (auto *Call = dyn_cast<CallBase>(I)) {
333 if (Call->isConvergent() || Call->cannotDuplicate()) {
334 LLVM_DEBUG(dbgs() << " Convergent call site found.\n");
335 return false;
336 }
337
Chandler Carruth363ac682019-01-07 05:42:51 +0000338 if (!AA->doesNotAccessMemory(Call)) {
339 LLVM_DEBUG(dbgs() << " Unsafe call site found.\n");
340 return false;
341 }
Matt Arsenault79b3ea72019-05-29 20:47:59 +0000342 }
343
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000344 // Avoid loops with possiblity of throw
345 if (I->mayThrow()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000346 LLVM_DEBUG(dbgs() << " May throw instruction found in loop body\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000347 return false;
348 }
349 // If current instruction is load instructions
350 // make sure it's a simple load (non atomic & non volatile)
351 if (I->mayReadFromMemory()) {
352 LoadInst *Ld = dyn_cast<LoadInst>(I);
353 if (!Ld || !Ld->isSimple()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000354 LLVM_DEBUG(dbgs() << " Found a non-simple load.\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000355 return false;
356 }
357 LoadAndStoreCounter++;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000358 Value *Ptr = Ld->getPointerOperand();
359 // Check loop invariant.
360 if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
361 InvariantCounter++;
362 }
363 // If current instruction is store instruction
364 // make sure it's a simple store (non atomic & non volatile)
365 else if (I->mayWriteToMemory()) {
366 StoreInst *St = dyn_cast<StoreInst>(I);
367 if (!St || !St->isSimple()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000368 LLVM_DEBUG(dbgs() << " Found a non-simple store.\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000369 return false;
370 }
371 LoadAndStoreCounter++;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000372 Value *Ptr = St->getPointerOperand();
373 // Check loop invariant.
374 if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop))
375 InvariantCounter++;
376
377 IsReadOnlyLoop = false;
378 }
379 return true;
380}
381
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000382/// Check loop instructions and confirms it's good for
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000383/// LoopVersioningLICM.
384bool LoopVersioningLICM::legalLoopInstructions() {
385 // Resetting counters.
386 LoadAndStoreCounter = 0;
387 InvariantCounter = 0;
388 IsReadOnlyLoop = true;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000389 using namespace ore;
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000390 // Iterate over loop blocks and instructions of each block and check
391 // instruction safety.
392 for (auto *Block : CurLoop->getBlocks())
393 for (auto &Inst : *Block) {
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000394 // If instruction is unsafe just return false.
Ashutosh Nema007b4252018-01-23 09:47:28 +0000395 if (!instructionSafeForVersioning(&Inst)) {
396 ORE->emit([&]() {
397 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopInst", &Inst)
398 << " Unsafe Loop Instruction";
399 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000400 return false;
Ashutosh Nema007b4252018-01-23 09:47:28 +0000401 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000402 }
403 // Get LoopAccessInfo from current loop.
Adam Nemeta9f09c62016-06-17 22:35:41 +0000404 LAI = &LAA->getInfo(CurLoop);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000405 // Check LoopAccessInfo for need of runtime check.
406 if (LAI->getRuntimePointerChecking()->getChecks().empty()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000407 LLVM_DEBUG(dbgs() << " LAA: Runtime check not found !!\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000408 return false;
409 }
410 // Number of runtime-checks should be less then RuntimeMemoryCheckThreshold
411 if (LAI->getNumRuntimePointerChecks() >
412 VectorizerParams::RuntimeMemoryCheckThreshold) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000413 LLVM_DEBUG(
414 dbgs() << " LAA: Runtime checks are more than threshold !!\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000415 ORE->emit([&]() {
416 return OptimizationRemarkMissed(DEBUG_TYPE, "RuntimeCheck",
417 CurLoop->getStartLoc(),
418 CurLoop->getHeader())
419 << "Number of runtime checks "
420 << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks())
421 << " exceeds threshold "
422 << NV("Threshold", VectorizerParams::RuntimeMemoryCheckThreshold);
423 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000424 return false;
425 }
426 // Loop should have at least one invariant load or store instruction.
427 if (!InvariantCounter) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000428 LLVM_DEBUG(dbgs() << " Invariant not found !!\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000429 return false;
430 }
431 // Read only loop not allowed.
432 if (IsReadOnlyLoop) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000433 LLVM_DEBUG(dbgs() << " Found a read-only loop!\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000434 return false;
435 }
436 // Profitablity check:
437 // Check invariant threshold, should be in limit.
438 if (InvariantCounter * 100 < InvariantThreshold * LoadAndStoreCounter) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000439 LLVM_DEBUG(
440 dbgs()
441 << " Invariant load & store are less then defined threshold\n");
442 LLVM_DEBUG(dbgs() << " Invariant loads & stores: "
443 << ((InvariantCounter * 100) / LoadAndStoreCounter)
444 << "%\n");
445 LLVM_DEBUG(dbgs() << " Invariant loads & store threshold: "
446 << InvariantThreshold << "%\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000447 ORE->emit([&]() {
448 return OptimizationRemarkMissed(DEBUG_TYPE, "InvariantThreshold",
449 CurLoop->getStartLoc(),
450 CurLoop->getHeader())
451 << "Invariant load & store "
452 << NV("LoadAndStoreCounter",
453 ((InvariantCounter * 100) / LoadAndStoreCounter))
454 << " are less then defined threshold "
455 << NV("Threshold", InvariantThreshold);
456 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000457 return false;
458 }
459 return true;
460}
461
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000462/// It checks loop is already visited or not.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000463/// check loop meta data, if loop revisited return true
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000464/// else false.
465bool LoopVersioningLICM::isLoopAlreadyVisited() {
466 // Check LoopVersioningLICM metadata into loop
Adam Nemetf7878262016-04-21 17:33:12 +0000467 if (findStringMetadataForLoop(CurLoop, LICMVersioningMetaData)) {
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000468 return true;
469 }
470 return false;
471}
472
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000473/// Checks legality for LoopVersioningLICM by considering following:
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000474/// a) loop structure legality b) loop instruction legality
475/// c) loop memory access legality.
476/// Return true if legal else returns false.
477bool LoopVersioningLICM::isLegalForVersioning() {
Ashutosh Nema007b4252018-01-23 09:47:28 +0000478 using namespace ore;
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000479 LLVM_DEBUG(dbgs() << "Loop: " << *CurLoop);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000480 // Make sure not re-visiting same loop again.
481 if (isLoopAlreadyVisited()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000482 LLVM_DEBUG(
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000483 dbgs() << " Revisiting loop in LoopVersioningLICM not allowed.\n\n");
484 return false;
485 }
486 // Check loop structure leagality.
487 if (!legalLoopStructure()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000488 LLVM_DEBUG(
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000489 dbgs() << " Loop structure not suitable for LoopVersioningLICM\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000490 ORE->emit([&]() {
491 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopStruct",
492 CurLoop->getStartLoc(),
493 CurLoop->getHeader())
494 << " Unsafe Loop structure";
495 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000496 return false;
497 }
498 // Check loop instruction leagality.
499 if (!legalLoopInstructions()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000500 LLVM_DEBUG(
501 dbgs()
502 << " Loop instructions not suitable for LoopVersioningLICM\n\n");
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000503 return false;
504 }
505 // Check loop memory access leagality.
506 if (!legalLoopMemoryAccesses()) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000507 LLVM_DEBUG(
508 dbgs()
509 << " Loop memory access not suitable for LoopVersioningLICM\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000510 ORE->emit([&]() {
511 return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopMemoryAccess",
512 CurLoop->getStartLoc(),
513 CurLoop->getHeader())
514 << " Unsafe Loop memory access";
515 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000516 return false;
517 }
518 // Loop versioning is feasible, return true.
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000519 LLVM_DEBUG(dbgs() << " Loop Versioning found to be beneficial\n\n");
Ashutosh Nema007b4252018-01-23 09:47:28 +0000520 ORE->emit([&]() {
521 return OptimizationRemark(DEBUG_TYPE, "IsLegalForVersioning",
522 CurLoop->getStartLoc(), CurLoop->getHeader())
523 << " Versioned loop for LICM."
524 << " Number of runtime checks we had to insert "
525 << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks());
526 });
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000527 return true;
528}
529
Adrian Prantl5f8f34e42018-05-01 15:54:18 +0000530/// Update loop with aggressive aliasing assumptions.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000531/// It marks no-alias to any pairs of memory operations by assuming
532/// loop should not have any must-alias memory accesses pairs.
533/// During LoopVersioningLICM legality we ignore loops having must
534/// aliasing memory accesses.
535void LoopVersioningLICM::setNoAliasToLoop(Loop *VerLoop) {
536 // Get latch terminator instruction.
537 Instruction *I = VerLoop->getLoopLatch()->getTerminator();
538 // Create alias scope domain.
539 MDBuilder MDB(I->getContext());
540 MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("LVDomain");
541 StringRef Name = "LVAliasScope";
542 SmallVector<Metadata *, 4> Scopes, NoAliases;
543 MDNode *NewScope = MDB.createAnonymousAliasScope(NewDomain, Name);
544 // Iterate over each instruction of loop.
545 // set no-alias for all load & store instructions.
546 for (auto *Block : CurLoop->getBlocks()) {
547 for (auto &Inst : *Block) {
548 // Only interested in instruction that may modify or read memory.
549 if (!Inst.mayReadFromMemory() && !Inst.mayWriteToMemory())
550 continue;
551 Scopes.push_back(NewScope);
552 NoAliases.push_back(NewScope);
553 // Set no-alias for current instruction.
554 Inst.setMetadata(
555 LLVMContext::MD_noalias,
556 MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_noalias),
557 MDNode::get(Inst.getContext(), NoAliases)));
558 // set alias-scope for current instruction.
559 Inst.setMetadata(
560 LLVMContext::MD_alias_scope,
561 MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_alias_scope),
562 MDNode::get(Inst.getContext(), Scopes)));
563 }
564 }
565}
566
567bool LoopVersioningLICM::runOnLoop(Loop *L, LPPassManager &LPM) {
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000568 // This will automatically release all resources hold by the current
569 // LoopVersioningLICM object.
570 AutoResetter Resetter(*this);
571
Andrew Kayloraa641a52016-04-22 22:06:11 +0000572 if (skipLoop(L))
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000573 return false;
Michael Kruse72448522018-12-12 17:32:52 +0000574
575 // Do not do the transformation if disabled by metadata.
576 if (hasLICMVersioningTransformation(L) & TM_Disable)
577 return false;
578
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000579 // Get Analysis information.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000580 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
581 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000582 LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
Ashutosh Nema007b4252018-01-23 09:47:28 +0000583 ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000584 LAI = nullptr;
585 // Set Current Loop
586 CurLoop = L;
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000587 CurAST.reset(new AliasSetTracker(*AA));
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000588
589 // Loop over the body of this loop, construct AST.
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000590 LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000591 for (auto *Block : L->getBlocks()) {
592 if (LI->getLoopFor(Block) == L) // Ignore blocks in subloop.
593 CurAST->add(*Block); // Incorporate the specified basic block
594 }
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000595
596 bool Changed = false;
597
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000598 // Check feasiblity of LoopVersioningLICM.
599 // If versioning found to be feasible and beneficial then proceed
600 // else simply return, by cleaning up memory.
601 if (isLegalForVersioning()) {
602 // Do loop versioning.
603 // Create memcheck for memory accessed inside loop.
604 // Clone original loop, and set blocks properly.
Evgeny Astigeevich48fd87e2016-10-14 23:00:36 +0000605 DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000606 LoopVersioning LVer(*LAI, CurLoop, LI, DT, SE, true);
607 LVer.versionLoop();
608 // Set Loop Versioning metaData for original loop.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000609 addStringMetadataToLoop(LVer.getNonVersionedLoop(), LICMVersioningMetaData);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000610 // Set Loop Versioning metaData for version loop.
Ashutosh Nema2260a3a2016-02-11 09:23:53 +0000611 addStringMetadataToLoop(LVer.getVersionedLoop(), LICMVersioningMetaData);
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000612 // Set "llvm.mem.parallel_loop_access" metaData to versioned loop.
Michael Kruse978ba612018-12-20 04:58:07 +0000613 // FIXME: "llvm.mem.parallel_loop_access" annotates memory access
614 // instructions, not loops.
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000615 addStringMetadataToLoop(LVer.getVersionedLoop(),
616 "llvm.mem.parallel_loop_access");
617 // Update version loop with aggressive aliasing assumption.
618 setNoAliasToLoop(LVer.getVersionedLoop());
619 Changed = true;
620 }
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000621 return Changed;
622}
623
624char LoopVersioningLICM::ID = 0;
Eugene Zelenko5adb96c2017-10-26 00:55:39 +0000625
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000626INITIALIZE_PASS_BEGIN(LoopVersioningLICM, "loop-versioning-licm",
627 "Loop Versioning For LICM", false, false)
628INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
629INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
630INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
Easwaran Ramane12c4872016-06-09 19:44:46 +0000631INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass)
Xinliang David Li7853c1d2016-07-08 20:55:26 +0000632INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000633INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
634INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
635INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
Ashutosh Nema007b4252018-01-23 09:47:28 +0000636INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
Ashutosh Nemadf6763a2016-02-06 07:47:48 +0000637INITIALIZE_PASS_END(LoopVersioningLICM, "loop-versioning-licm",
638 "Loop Versioning For LICM", false, false)
639
640Pass *llvm::createLoopVersioningLICMPass() { return new LoopVersioningLICM(); }