Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 1 | //===- LoopVersioningLICM.cpp - LICM Loop Versioning ----------------------===// |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 2 | // |
Chandler Carruth | 2946cd7 | 2019-01-19 08:50:56 +0000 | [diff] [blame] | 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // When alias analysis is uncertain about the aliasing between any two accesses, |
| 10 | // it will return MayAlias. This uncertainty from alias analysis restricts LICM |
| 11 | // from proceeding further. In cases where alias analysis is uncertain we might |
| 12 | // use loop versioning as an alternative. |
| 13 | // |
| 14 | // Loop Versioning will create a version of the loop with aggressive aliasing |
| 15 | // assumptions in addition to the original with conservative (default) aliasing |
| 16 | // assumptions. The version of the loop making aggressive aliasing assumptions |
| 17 | // will have all the memory accesses marked as no-alias. These two versions of |
| 18 | // loop will be preceded by a memory runtime check. This runtime check consists |
| 19 | // of bound checks for all unique memory accessed in loop, and it ensures the |
| 20 | // lack of memory aliasing. The result of the runtime check determines which of |
| 21 | // the loop versions is executed: If the runtime check detects any memory |
| 22 | // aliasing, then the original loop is executed. Otherwise, the version with |
| 23 | // aggressive aliasing assumptions is used. |
| 24 | // |
| 25 | // Following are the top level steps: |
| 26 | // |
| 27 | // a) Perform LoopVersioningLICM's feasibility check. |
| 28 | // b) If loop is a candidate for versioning then create a memory bound check, |
| 29 | // by considering all the memory accesses in loop body. |
| 30 | // c) Clone original loop and set all memory accesses as no-alias in new loop. |
| 31 | // d) Set original loop & versioned loop as a branch target of the runtime check |
| 32 | // result. |
| 33 | // |
| 34 | // It transforms loop as shown below: |
| 35 | // |
| 36 | // +----------------+ |
| 37 | // |Runtime Memcheck| |
| 38 | // +----------------+ |
| 39 | // | |
| 40 | // +----------+----------------+----------+ |
| 41 | // | | |
| 42 | // +---------+----------+ +-----------+----------+ |
| 43 | // |Orig Loop Preheader | |Cloned Loop Preheader | |
| 44 | // +--------------------+ +----------------------+ |
| 45 | // | | |
| 46 | // +--------------------+ +----------------------+ |
| 47 | // |Orig Loop Body | |Cloned Loop Body | |
| 48 | // +--------------------+ +----------------------+ |
| 49 | // | | |
| 50 | // +--------------------+ +----------------------+ |
Ashutosh Nema | 2260a3a | 2016-02-11 09:23:53 +0000 | [diff] [blame] | 51 | // |Orig Loop Exit Block| |Cloned Loop Exit Block| |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 52 | // +--------------------+ +-----------+----------+ |
| 53 | // | | |
| 54 | // +----------+--------------+-----------+ |
| 55 | // | |
| 56 | // +-----+----+ |
| 57 | // |Join Block| |
| 58 | // +----------+ |
| 59 | // |
| 60 | //===----------------------------------------------------------------------===// |
| 61 | |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 62 | #include "llvm/ADT/SmallVector.h" |
| 63 | #include "llvm/ADT/StringRef.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 64 | #include "llvm/Analysis/AliasAnalysis.h" |
| 65 | #include "llvm/Analysis/AliasSetTracker.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 66 | #include "llvm/Analysis/GlobalsModRef.h" |
| 67 | #include "llvm/Analysis/LoopAccessAnalysis.h" |
| 68 | #include "llvm/Analysis/LoopInfo.h" |
| 69 | #include "llvm/Analysis/LoopPass.h" |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 70 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 71 | #include "llvm/Analysis/ScalarEvolution.h" |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 72 | #include "llvm/IR/CallSite.h" |
| 73 | #include "llvm/IR/Constants.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 74 | #include "llvm/IR/Dominators.h" |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 75 | #include "llvm/IR/Instruction.h" |
| 76 | #include "llvm/IR/Instructions.h" |
| 77 | #include "llvm/IR/LLVMContext.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 78 | #include "llvm/IR/MDBuilder.h" |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 79 | #include "llvm/IR/Metadata.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 80 | #include "llvm/IR/Type.h" |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 81 | #include "llvm/IR/Value.h" |
| 82 | #include "llvm/Pass.h" |
| 83 | #include "llvm/Support/Casting.h" |
| 84 | #include "llvm/Support/CommandLine.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 85 | #include "llvm/Support/Debug.h" |
| 86 | #include "llvm/Support/raw_ostream.h" |
| 87 | #include "llvm/Transforms/Scalar.h" |
David Blaikie | a373d18 | 2018-03-28 17:44:36 +0000 | [diff] [blame] | 88 | #include "llvm/Transforms/Utils.h" |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 89 | #include "llvm/Transforms/Utils/LoopUtils.h" |
| 90 | #include "llvm/Transforms/Utils/LoopVersioning.h" |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 91 | #include <cassert> |
| 92 | #include <memory> |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 93 | |
| 94 | using namespace llvm; |
| 95 | |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 96 | #define DEBUG_TYPE "loop-versioning-licm" |
| 97 | |
| 98 | static const char *LICMVersioningMetaData = "llvm.loop.licm_versioning.disable"; |
| 99 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 100 | /// Threshold minimum allowed percentage for possible |
| 101 | /// invariant instructions in a loop. |
| 102 | static cl::opt<float> |
Ashutosh Nema | 2260a3a | 2016-02-11 09:23:53 +0000 | [diff] [blame] | 103 | LVInvarThreshold("licm-versioning-invariant-threshold", |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 104 | cl::desc("LoopVersioningLICM's minimum allowed percentage" |
| 105 | "of possible invariant instructions per loop"), |
| 106 | cl::init(25), cl::Hidden); |
| 107 | |
| 108 | /// Threshold for maximum allowed loop nest/depth |
| 109 | static cl::opt<unsigned> LVLoopDepthThreshold( |
Ashutosh Nema | 2260a3a | 2016-02-11 09:23:53 +0000 | [diff] [blame] | 110 | "licm-versioning-max-depth-threshold", |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 111 | cl::desc( |
| 112 | "LoopVersioningLICM's threshold for maximum allowed loop nest/depth"), |
| 113 | cl::init(2), cl::Hidden); |
| 114 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 115 | namespace { |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 116 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 117 | struct LoopVersioningLICM : public LoopPass { |
| 118 | static char ID; |
| 119 | |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 120 | LoopVersioningLICM() |
| 121 | : LoopPass(ID), LoopDepthThreshold(LVLoopDepthThreshold), |
| 122 | InvariantThreshold(LVInvarThreshold) { |
| 123 | initializeLoopVersioningLICMPass(*PassRegistry::getPassRegistry()); |
| 124 | } |
| 125 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 126 | bool runOnLoop(Loop *L, LPPassManager &LPM) override; |
| 127 | |
| 128 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 129 | AU.setPreservesCFG(); |
| 130 | AU.addRequired<AAResultsWrapperPass>(); |
| 131 | AU.addRequired<DominatorTreeWrapperPass>(); |
| 132 | AU.addRequiredID(LCSSAID); |
Xinliang David Li | 7853c1d | 2016-07-08 20:55:26 +0000 | [diff] [blame] | 133 | AU.addRequired<LoopAccessLegacyAnalysis>(); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 134 | AU.addRequired<LoopInfoWrapperPass>(); |
| 135 | AU.addRequiredID(LoopSimplifyID); |
| 136 | AU.addRequired<ScalarEvolutionWrapperPass>(); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 137 | AU.addPreserved<AAResultsWrapperPass>(); |
| 138 | AU.addPreserved<GlobalsAAWrapperPass>(); |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 139 | AU.addRequired<OptimizationRemarkEmitterWrapperPass>(); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 140 | } |
| 141 | |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 142 | StringRef getPassName() const override { return "Loop Versioning for LICM"; } |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 143 | |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 144 | void reset() { |
| 145 | AA = nullptr; |
| 146 | SE = nullptr; |
| 147 | LAA = nullptr; |
| 148 | CurLoop = nullptr; |
| 149 | LoadAndStoreCounter = 0; |
| 150 | InvariantCounter = 0; |
| 151 | IsReadOnlyLoop = true; |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 152 | ORE = nullptr; |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 153 | CurAST.reset(); |
| 154 | } |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 155 | |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 156 | class AutoResetter { |
| 157 | public: |
| 158 | AutoResetter(LoopVersioningLICM &LVLICM) : LVLICM(LVLICM) {} |
| 159 | ~AutoResetter() { LVLICM.reset(); } |
| 160 | |
| 161 | private: |
| 162 | LoopVersioningLICM &LVLICM; |
| 163 | }; |
| 164 | |
| 165 | private: |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 166 | // Current AliasAnalysis information |
| 167 | AliasAnalysis *AA = nullptr; |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 168 | |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 169 | // Current ScalarEvolution |
| 170 | ScalarEvolution *SE = nullptr; |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 171 | |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 172 | // Current LoopAccessAnalysis |
| 173 | LoopAccessLegacyAnalysis *LAA = nullptr; |
| 174 | |
| 175 | // Current Loop's LoopAccessInfo |
| 176 | const LoopAccessInfo *LAI = nullptr; |
| 177 | |
| 178 | // The current loop we are working on. |
| 179 | Loop *CurLoop = nullptr; |
| 180 | |
| 181 | // AliasSet information for the current loop. |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 182 | std::unique_ptr<AliasSetTracker> CurAST; |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 183 | |
| 184 | // Maximum loop nest threshold |
| 185 | unsigned LoopDepthThreshold; |
| 186 | |
| 187 | // Minimum invariant threshold |
| 188 | float InvariantThreshold; |
| 189 | |
| 190 | // Counter to track num of load & store |
| 191 | unsigned LoadAndStoreCounter = 0; |
| 192 | |
| 193 | // Counter to track num of invariant |
| 194 | unsigned InvariantCounter = 0; |
| 195 | |
| 196 | // Read only loop marker. |
| 197 | bool IsReadOnlyLoop = true; |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 198 | |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 199 | // OptimizationRemarkEmitter |
| 200 | OptimizationRemarkEmitter *ORE; |
| 201 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 202 | bool isLegalForVersioning(); |
| 203 | bool legalLoopStructure(); |
| 204 | bool legalLoopInstructions(); |
| 205 | bool legalLoopMemoryAccesses(); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 206 | bool isLoopAlreadyVisited(); |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 207 | void setNoAliasToLoop(Loop *VerLoop); |
| 208 | bool instructionSafeForVersioning(Instruction *I); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 209 | }; |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 210 | |
| 211 | } // end anonymous namespace |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 212 | |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 213 | /// Check loop structure and confirms it's good for LoopVersioningLICM. |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 214 | bool LoopVersioningLICM::legalLoopStructure() { |
Florian Hahn | 2e03213 | 2016-12-19 17:13:37 +0000 | [diff] [blame] | 215 | // Loop must be in loop simplify form. |
| 216 | if (!CurLoop->isLoopSimplifyForm()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 217 | LLVM_DEBUG(dbgs() << " loop is not in loop-simplify form.\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 218 | return false; |
| 219 | } |
| 220 | // Loop should be innermost loop, if not return false. |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 221 | if (!CurLoop->getSubLoops().empty()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 222 | LLVM_DEBUG(dbgs() << " loop is not innermost\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 223 | return false; |
| 224 | } |
| 225 | // Loop should have a single backedge, if not return false. |
| 226 | if (CurLoop->getNumBackEdges() != 1) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 227 | LLVM_DEBUG(dbgs() << " loop has multiple backedges\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 228 | return false; |
| 229 | } |
| 230 | // Loop must have a single exiting block, if not return false. |
| 231 | if (!CurLoop->getExitingBlock()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 232 | LLVM_DEBUG(dbgs() << " loop has multiple exiting block\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 233 | return false; |
| 234 | } |
| 235 | // We only handle bottom-tested loop, i.e. loop in which the condition is |
| 236 | // checked at the end of each iteration. With that we can assume that all |
| 237 | // instructions in the loop are executed the same number of times. |
| 238 | if (CurLoop->getExitingBlock() != CurLoop->getLoopLatch()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 239 | LLVM_DEBUG(dbgs() << " loop is not bottom tested\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 240 | return false; |
| 241 | } |
| 242 | // Parallel loops must not have aliasing loop-invariant memory accesses. |
| 243 | // Hence we don't need to version anything in this case. |
| 244 | if (CurLoop->isAnnotatedParallel()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 245 | LLVM_DEBUG(dbgs() << " Parallel loop is not worth versioning\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 246 | return false; |
| 247 | } |
| 248 | // Loop depth more then LoopDepthThreshold are not allowed |
| 249 | if (CurLoop->getLoopDepth() > LoopDepthThreshold) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 250 | LLVM_DEBUG(dbgs() << " loop depth is more then threshold\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 251 | return false; |
| 252 | } |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 253 | // We need to be able to compute the loop trip count in order |
| 254 | // to generate the bound checks. |
| 255 | const SCEV *ExitCount = SE->getBackedgeTakenCount(CurLoop); |
| 256 | if (ExitCount == SE->getCouldNotCompute()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 257 | LLVM_DEBUG(dbgs() << " loop does not has trip count\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 258 | return false; |
| 259 | } |
| 260 | return true; |
| 261 | } |
| 262 | |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 263 | /// Check memory accesses in loop and confirms it's good for |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 264 | /// LoopVersioningLICM. |
| 265 | bool LoopVersioningLICM::legalLoopMemoryAccesses() { |
| 266 | bool HasMayAlias = false; |
| 267 | bool TypeSafety = false; |
| 268 | bool HasMod = false; |
| 269 | // Memory check: |
| 270 | // Transform phase will generate a versioned loop and also a runtime check to |
| 271 | // ensure the pointers are independent and they don’t alias. |
| 272 | // In version variant of loop, alias meta data asserts that all access are |
| 273 | // mutually independent. |
| 274 | // |
| 275 | // Pointers aliasing in alias domain are avoided because with multiple |
| 276 | // aliasing domains we may not be able to hoist potential loop invariant |
| 277 | // access out of the loop. |
| 278 | // |
| 279 | // Iterate over alias tracker sets, and confirm AliasSets doesn't have any |
| 280 | // must alias set. |
| 281 | for (const auto &I : *CurAST) { |
| 282 | const AliasSet &AS = I; |
| 283 | // Skip Forward Alias Sets, as this should be ignored as part of |
| 284 | // the AliasSetTracker object. |
| 285 | if (AS.isForwardingAliasSet()) |
| 286 | continue; |
| 287 | // With MustAlias its not worth adding runtime bound check. |
| 288 | if (AS.isMustAlias()) |
| 289 | return false; |
| 290 | Value *SomePtr = AS.begin()->getValue(); |
| 291 | bool TypeCheck = true; |
| 292 | // Check for Mod & MayAlias |
| 293 | HasMayAlias |= AS.isMayAlias(); |
| 294 | HasMod |= AS.isMod(); |
| 295 | for (const auto &A : AS) { |
| 296 | Value *Ptr = A.getValue(); |
| 297 | // Alias tracker should have pointers of same data type. |
| 298 | TypeCheck = (TypeCheck && (SomePtr->getType() == Ptr->getType())); |
| 299 | } |
| 300 | // At least one alias tracker should have pointers of same data type. |
| 301 | TypeSafety |= TypeCheck; |
| 302 | } |
| 303 | // Ensure types should be of same type. |
| 304 | if (!TypeSafety) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 305 | LLVM_DEBUG(dbgs() << " Alias tracker type safety failed!\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 306 | return false; |
| 307 | } |
| 308 | // Ensure loop body shouldn't be read only. |
| 309 | if (!HasMod) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 310 | LLVM_DEBUG(dbgs() << " No memory modified in loop body\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 311 | return false; |
| 312 | } |
| 313 | // Make sure alias set has may alias case. |
| 314 | // If there no alias memory ambiguity, return false. |
| 315 | if (!HasMayAlias) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 316 | LLVM_DEBUG(dbgs() << " No ambiguity in memory access.\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 317 | return false; |
| 318 | } |
| 319 | return true; |
| 320 | } |
| 321 | |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 322 | /// Check loop instructions safe for Loop versioning. |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 323 | /// It returns true if it's safe else returns false. |
| 324 | /// Consider following: |
| 325 | /// 1) Check all load store in loop body are non atomic & non volatile. |
| 326 | /// 2) Check function call safety, by ensuring its not accessing memory. |
| 327 | /// 3) Loop body shouldn't have any may throw instruction. |
Matt Arsenault | 79b3ea7 | 2019-05-29 20:47:59 +0000 | [diff] [blame] | 328 | /// 4) Loop body shouldn't have any convergent or noduplicate instructions. |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 329 | bool LoopVersioningLICM::instructionSafeForVersioning(Instruction *I) { |
| 330 | assert(I != nullptr && "Null instruction found!"); |
| 331 | // Check function call safety |
Matt Arsenault | 79b3ea7 | 2019-05-29 20:47:59 +0000 | [diff] [blame] | 332 | if (auto *Call = dyn_cast<CallBase>(I)) { |
| 333 | if (Call->isConvergent() || Call->cannotDuplicate()) { |
| 334 | LLVM_DEBUG(dbgs() << " Convergent call site found.\n"); |
| 335 | return false; |
| 336 | } |
| 337 | |
Chandler Carruth | 363ac68 | 2019-01-07 05:42:51 +0000 | [diff] [blame] | 338 | if (!AA->doesNotAccessMemory(Call)) { |
| 339 | LLVM_DEBUG(dbgs() << " Unsafe call site found.\n"); |
| 340 | return false; |
| 341 | } |
Matt Arsenault | 79b3ea7 | 2019-05-29 20:47:59 +0000 | [diff] [blame] | 342 | } |
| 343 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 344 | // Avoid loops with possiblity of throw |
| 345 | if (I->mayThrow()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 346 | LLVM_DEBUG(dbgs() << " May throw instruction found in loop body\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 347 | return false; |
| 348 | } |
| 349 | // If current instruction is load instructions |
| 350 | // make sure it's a simple load (non atomic & non volatile) |
| 351 | if (I->mayReadFromMemory()) { |
| 352 | LoadInst *Ld = dyn_cast<LoadInst>(I); |
| 353 | if (!Ld || !Ld->isSimple()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 354 | LLVM_DEBUG(dbgs() << " Found a non-simple load.\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 355 | return false; |
| 356 | } |
| 357 | LoadAndStoreCounter++; |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 358 | Value *Ptr = Ld->getPointerOperand(); |
| 359 | // Check loop invariant. |
| 360 | if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop)) |
| 361 | InvariantCounter++; |
| 362 | } |
| 363 | // If current instruction is store instruction |
| 364 | // make sure it's a simple store (non atomic & non volatile) |
| 365 | else if (I->mayWriteToMemory()) { |
| 366 | StoreInst *St = dyn_cast<StoreInst>(I); |
| 367 | if (!St || !St->isSimple()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 368 | LLVM_DEBUG(dbgs() << " Found a non-simple store.\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 369 | return false; |
| 370 | } |
| 371 | LoadAndStoreCounter++; |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 372 | Value *Ptr = St->getPointerOperand(); |
| 373 | // Check loop invariant. |
| 374 | if (SE->isLoopInvariant(SE->getSCEV(Ptr), CurLoop)) |
| 375 | InvariantCounter++; |
| 376 | |
| 377 | IsReadOnlyLoop = false; |
| 378 | } |
| 379 | return true; |
| 380 | } |
| 381 | |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 382 | /// Check loop instructions and confirms it's good for |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 383 | /// LoopVersioningLICM. |
| 384 | bool LoopVersioningLICM::legalLoopInstructions() { |
| 385 | // Resetting counters. |
| 386 | LoadAndStoreCounter = 0; |
| 387 | InvariantCounter = 0; |
| 388 | IsReadOnlyLoop = true; |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 389 | using namespace ore; |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 390 | // Iterate over loop blocks and instructions of each block and check |
| 391 | // instruction safety. |
| 392 | for (auto *Block : CurLoop->getBlocks()) |
| 393 | for (auto &Inst : *Block) { |
Ashutosh Nema | 2260a3a | 2016-02-11 09:23:53 +0000 | [diff] [blame] | 394 | // If instruction is unsafe just return false. |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 395 | if (!instructionSafeForVersioning(&Inst)) { |
| 396 | ORE->emit([&]() { |
| 397 | return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopInst", &Inst) |
| 398 | << " Unsafe Loop Instruction"; |
| 399 | }); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 400 | return false; |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 401 | } |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 402 | } |
| 403 | // Get LoopAccessInfo from current loop. |
Adam Nemet | a9f09c6 | 2016-06-17 22:35:41 +0000 | [diff] [blame] | 404 | LAI = &LAA->getInfo(CurLoop); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 405 | // Check LoopAccessInfo for need of runtime check. |
| 406 | if (LAI->getRuntimePointerChecking()->getChecks().empty()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 407 | LLVM_DEBUG(dbgs() << " LAA: Runtime check not found !!\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 408 | return false; |
| 409 | } |
| 410 | // Number of runtime-checks should be less then RuntimeMemoryCheckThreshold |
| 411 | if (LAI->getNumRuntimePointerChecks() > |
| 412 | VectorizerParams::RuntimeMemoryCheckThreshold) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 413 | LLVM_DEBUG( |
| 414 | dbgs() << " LAA: Runtime checks are more than threshold !!\n"); |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 415 | ORE->emit([&]() { |
| 416 | return OptimizationRemarkMissed(DEBUG_TYPE, "RuntimeCheck", |
| 417 | CurLoop->getStartLoc(), |
| 418 | CurLoop->getHeader()) |
| 419 | << "Number of runtime checks " |
| 420 | << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks()) |
| 421 | << " exceeds threshold " |
| 422 | << NV("Threshold", VectorizerParams::RuntimeMemoryCheckThreshold); |
| 423 | }); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 424 | return false; |
| 425 | } |
| 426 | // Loop should have at least one invariant load or store instruction. |
| 427 | if (!InvariantCounter) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 428 | LLVM_DEBUG(dbgs() << " Invariant not found !!\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 429 | return false; |
| 430 | } |
| 431 | // Read only loop not allowed. |
| 432 | if (IsReadOnlyLoop) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 433 | LLVM_DEBUG(dbgs() << " Found a read-only loop!\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 434 | return false; |
| 435 | } |
| 436 | // Profitablity check: |
| 437 | // Check invariant threshold, should be in limit. |
| 438 | if (InvariantCounter * 100 < InvariantThreshold * LoadAndStoreCounter) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 439 | LLVM_DEBUG( |
| 440 | dbgs() |
| 441 | << " Invariant load & store are less then defined threshold\n"); |
| 442 | LLVM_DEBUG(dbgs() << " Invariant loads & stores: " |
| 443 | << ((InvariantCounter * 100) / LoadAndStoreCounter) |
| 444 | << "%\n"); |
| 445 | LLVM_DEBUG(dbgs() << " Invariant loads & store threshold: " |
| 446 | << InvariantThreshold << "%\n"); |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 447 | ORE->emit([&]() { |
| 448 | return OptimizationRemarkMissed(DEBUG_TYPE, "InvariantThreshold", |
| 449 | CurLoop->getStartLoc(), |
| 450 | CurLoop->getHeader()) |
| 451 | << "Invariant load & store " |
| 452 | << NV("LoadAndStoreCounter", |
| 453 | ((InvariantCounter * 100) / LoadAndStoreCounter)) |
| 454 | << " are less then defined threshold " |
| 455 | << NV("Threshold", InvariantThreshold); |
| 456 | }); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 457 | return false; |
| 458 | } |
| 459 | return true; |
| 460 | } |
| 461 | |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 462 | /// It checks loop is already visited or not. |
Ashutosh Nema | 2260a3a | 2016-02-11 09:23:53 +0000 | [diff] [blame] | 463 | /// check loop meta data, if loop revisited return true |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 464 | /// else false. |
| 465 | bool LoopVersioningLICM::isLoopAlreadyVisited() { |
| 466 | // Check LoopVersioningLICM metadata into loop |
Adam Nemet | f787826 | 2016-04-21 17:33:12 +0000 | [diff] [blame] | 467 | if (findStringMetadataForLoop(CurLoop, LICMVersioningMetaData)) { |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 468 | return true; |
| 469 | } |
| 470 | return false; |
| 471 | } |
| 472 | |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 473 | /// Checks legality for LoopVersioningLICM by considering following: |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 474 | /// a) loop structure legality b) loop instruction legality |
| 475 | /// c) loop memory access legality. |
| 476 | /// Return true if legal else returns false. |
| 477 | bool LoopVersioningLICM::isLegalForVersioning() { |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 478 | using namespace ore; |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 479 | LLVM_DEBUG(dbgs() << "Loop: " << *CurLoop); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 480 | // Make sure not re-visiting same loop again. |
| 481 | if (isLoopAlreadyVisited()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 482 | LLVM_DEBUG( |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 483 | dbgs() << " Revisiting loop in LoopVersioningLICM not allowed.\n\n"); |
| 484 | return false; |
| 485 | } |
| 486 | // Check loop structure leagality. |
| 487 | if (!legalLoopStructure()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 488 | LLVM_DEBUG( |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 489 | dbgs() << " Loop structure not suitable for LoopVersioningLICM\n\n"); |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 490 | ORE->emit([&]() { |
| 491 | return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopStruct", |
| 492 | CurLoop->getStartLoc(), |
| 493 | CurLoop->getHeader()) |
| 494 | << " Unsafe Loop structure"; |
| 495 | }); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 496 | return false; |
| 497 | } |
| 498 | // Check loop instruction leagality. |
| 499 | if (!legalLoopInstructions()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 500 | LLVM_DEBUG( |
| 501 | dbgs() |
| 502 | << " Loop instructions not suitable for LoopVersioningLICM\n\n"); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 503 | return false; |
| 504 | } |
| 505 | // Check loop memory access leagality. |
| 506 | if (!legalLoopMemoryAccesses()) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 507 | LLVM_DEBUG( |
| 508 | dbgs() |
| 509 | << " Loop memory access not suitable for LoopVersioningLICM\n\n"); |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 510 | ORE->emit([&]() { |
| 511 | return OptimizationRemarkMissed(DEBUG_TYPE, "IllegalLoopMemoryAccess", |
| 512 | CurLoop->getStartLoc(), |
| 513 | CurLoop->getHeader()) |
| 514 | << " Unsafe Loop memory access"; |
| 515 | }); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 516 | return false; |
| 517 | } |
| 518 | // Loop versioning is feasible, return true. |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 519 | LLVM_DEBUG(dbgs() << " Loop Versioning found to be beneficial\n\n"); |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 520 | ORE->emit([&]() { |
| 521 | return OptimizationRemark(DEBUG_TYPE, "IsLegalForVersioning", |
| 522 | CurLoop->getStartLoc(), CurLoop->getHeader()) |
| 523 | << " Versioned loop for LICM." |
| 524 | << " Number of runtime checks we had to insert " |
| 525 | << NV("RuntimeChecks", LAI->getNumRuntimePointerChecks()); |
| 526 | }); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 527 | return true; |
| 528 | } |
| 529 | |
Adrian Prantl | 5f8f34e4 | 2018-05-01 15:54:18 +0000 | [diff] [blame] | 530 | /// Update loop with aggressive aliasing assumptions. |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 531 | /// It marks no-alias to any pairs of memory operations by assuming |
| 532 | /// loop should not have any must-alias memory accesses pairs. |
| 533 | /// During LoopVersioningLICM legality we ignore loops having must |
| 534 | /// aliasing memory accesses. |
| 535 | void LoopVersioningLICM::setNoAliasToLoop(Loop *VerLoop) { |
| 536 | // Get latch terminator instruction. |
| 537 | Instruction *I = VerLoop->getLoopLatch()->getTerminator(); |
| 538 | // Create alias scope domain. |
| 539 | MDBuilder MDB(I->getContext()); |
| 540 | MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("LVDomain"); |
| 541 | StringRef Name = "LVAliasScope"; |
| 542 | SmallVector<Metadata *, 4> Scopes, NoAliases; |
| 543 | MDNode *NewScope = MDB.createAnonymousAliasScope(NewDomain, Name); |
| 544 | // Iterate over each instruction of loop. |
| 545 | // set no-alias for all load & store instructions. |
| 546 | for (auto *Block : CurLoop->getBlocks()) { |
| 547 | for (auto &Inst : *Block) { |
| 548 | // Only interested in instruction that may modify or read memory. |
| 549 | if (!Inst.mayReadFromMemory() && !Inst.mayWriteToMemory()) |
| 550 | continue; |
| 551 | Scopes.push_back(NewScope); |
| 552 | NoAliases.push_back(NewScope); |
| 553 | // Set no-alias for current instruction. |
| 554 | Inst.setMetadata( |
| 555 | LLVMContext::MD_noalias, |
| 556 | MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_noalias), |
| 557 | MDNode::get(Inst.getContext(), NoAliases))); |
| 558 | // set alias-scope for current instruction. |
| 559 | Inst.setMetadata( |
| 560 | LLVMContext::MD_alias_scope, |
| 561 | MDNode::concatenate(Inst.getMetadata(LLVMContext::MD_alias_scope), |
| 562 | MDNode::get(Inst.getContext(), Scopes))); |
| 563 | } |
| 564 | } |
| 565 | } |
| 566 | |
| 567 | bool LoopVersioningLICM::runOnLoop(Loop *L, LPPassManager &LPM) { |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 568 | // This will automatically release all resources hold by the current |
| 569 | // LoopVersioningLICM object. |
| 570 | AutoResetter Resetter(*this); |
| 571 | |
Andrew Kaylor | aa641a5 | 2016-04-22 22:06:11 +0000 | [diff] [blame] | 572 | if (skipLoop(L)) |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 573 | return false; |
Michael Kruse | 7244852 | 2018-12-12 17:32:52 +0000 | [diff] [blame] | 574 | |
| 575 | // Do not do the transformation if disabled by metadata. |
| 576 | if (hasLICMVersioningTransformation(L) & TM_Disable) |
| 577 | return false; |
| 578 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 579 | // Get Analysis information. |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 580 | AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); |
| 581 | SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE(); |
Xinliang David Li | 7853c1d | 2016-07-08 20:55:26 +0000 | [diff] [blame] | 582 | LAA = &getAnalysis<LoopAccessLegacyAnalysis>(); |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 583 | ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 584 | LAI = nullptr; |
| 585 | // Set Current Loop |
| 586 | CurLoop = L; |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 587 | CurAST.reset(new AliasSetTracker(*AA)); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 588 | |
| 589 | // Loop over the body of this loop, construct AST. |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 590 | LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 591 | for (auto *Block : L->getBlocks()) { |
| 592 | if (LI->getLoopFor(Block) == L) // Ignore blocks in subloop. |
| 593 | CurAST->add(*Block); // Incorporate the specified basic block |
| 594 | } |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 595 | |
| 596 | bool Changed = false; |
| 597 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 598 | // Check feasiblity of LoopVersioningLICM. |
| 599 | // If versioning found to be feasible and beneficial then proceed |
| 600 | // else simply return, by cleaning up memory. |
| 601 | if (isLegalForVersioning()) { |
| 602 | // Do loop versioning. |
| 603 | // Create memcheck for memory accessed inside loop. |
| 604 | // Clone original loop, and set blocks properly. |
Evgeny Astigeevich | 48fd87e | 2016-10-14 23:00:36 +0000 | [diff] [blame] | 605 | DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 606 | LoopVersioning LVer(*LAI, CurLoop, LI, DT, SE, true); |
| 607 | LVer.versionLoop(); |
| 608 | // Set Loop Versioning metaData for original loop. |
Ashutosh Nema | 2260a3a | 2016-02-11 09:23:53 +0000 | [diff] [blame] | 609 | addStringMetadataToLoop(LVer.getNonVersionedLoop(), LICMVersioningMetaData); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 610 | // Set Loop Versioning metaData for version loop. |
Ashutosh Nema | 2260a3a | 2016-02-11 09:23:53 +0000 | [diff] [blame] | 611 | addStringMetadataToLoop(LVer.getVersionedLoop(), LICMVersioningMetaData); |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 612 | // Set "llvm.mem.parallel_loop_access" metaData to versioned loop. |
Michael Kruse | 978ba61 | 2018-12-20 04:58:07 +0000 | [diff] [blame] | 613 | // FIXME: "llvm.mem.parallel_loop_access" annotates memory access |
| 614 | // instructions, not loops. |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 615 | addStringMetadataToLoop(LVer.getVersionedLoop(), |
| 616 | "llvm.mem.parallel_loop_access"); |
| 617 | // Update version loop with aggressive aliasing assumption. |
| 618 | setNoAliasToLoop(LVer.getVersionedLoop()); |
| 619 | Changed = true; |
| 620 | } |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 621 | return Changed; |
| 622 | } |
| 623 | |
| 624 | char LoopVersioningLICM::ID = 0; |
Eugene Zelenko | 5adb96c | 2017-10-26 00:55:39 +0000 | [diff] [blame] | 625 | |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 626 | INITIALIZE_PASS_BEGIN(LoopVersioningLICM, "loop-versioning-licm", |
| 627 | "Loop Versioning For LICM", false, false) |
| 628 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) |
| 629 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) |
| 630 | INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass) |
Easwaran Raman | e12c487 | 2016-06-09 19:44:46 +0000 | [diff] [blame] | 631 | INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass) |
Xinliang David Li | 7853c1d | 2016-07-08 20:55:26 +0000 | [diff] [blame] | 632 | INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis) |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 633 | INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) |
| 634 | INITIALIZE_PASS_DEPENDENCY(LoopSimplify) |
| 635 | INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) |
Ashutosh Nema | 007b425 | 2018-01-23 09:47:28 +0000 | [diff] [blame] | 636 | INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) |
Ashutosh Nema | df6763a | 2016-02-06 07:47:48 +0000 | [diff] [blame] | 637 | INITIALIZE_PASS_END(LoopVersioningLICM, "loop-versioning-licm", |
| 638 | "Loop Versioning For LICM", false, false) |
| 639 | |
| 640 | Pass *llvm::createLoopVersioningLICMPass() { return new LoopVersioningLICM(); } |