Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 1 | //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | |
| 10 | #include "CGLoopInfo.h" |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 11 | #include "clang/AST/ASTContext.h" |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 12 | #include "clang/AST/Attr.h" |
| 13 | #include "clang/Sema/LoopHint.h" |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 14 | #include "llvm/IR/BasicBlock.h" |
| 15 | #include "llvm/IR/Constants.h" |
| 16 | #include "llvm/IR/InstrTypes.h" |
| 17 | #include "llvm/IR/Instructions.h" |
| 18 | #include "llvm/IR/Metadata.h" |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 19 | using namespace clang::CodeGen; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 20 | using namespace llvm; |
| 21 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 22 | static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, |
| 23 | llvm::DebugLoc Location) { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 24 | |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 25 | if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && |
| 26 | Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && |
| 27 | Attrs.VectorizeEnable == LoopAttributes::Unspecified && |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 28 | Attrs.UnrollEnable == LoopAttributes::Unspecified && |
| 29 | Attrs.DistributeEnable == LoopAttributes::Unspecified && |
| 30 | !Location) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 31 | return nullptr; |
| 32 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 33 | SmallVector<Metadata *, 4> Args; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 34 | // Reserve operand 0 for loop id self reference. |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 35 | auto TempNode = MDNode::getTemporary(Ctx, None); |
| 36 | Args.push_back(TempNode.get()); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 37 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 38 | // If we have a valid debug location for the loop, add it. |
| 39 | if (Location) |
| 40 | Args.push_back(Location.getAsMDNode()); |
| 41 | |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 42 | // Setting vectorize.width |
| 43 | if (Attrs.VectorizeWidth > 0) { |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 44 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), |
| 45 | ConstantAsMetadata::get(ConstantInt::get( |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 46 | Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 47 | Args.push_back(MDNode::get(Ctx, Vals)); |
| 48 | } |
| 49 | |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 50 | // Setting interleave.count |
| 51 | if (Attrs.InterleaveCount > 0) { |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 52 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), |
| 53 | ConstantAsMetadata::get(ConstantInt::get( |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 54 | Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 55 | Args.push_back(MDNode::get(Ctx, Vals)); |
| 56 | } |
| 57 | |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 58 | // Setting interleave.count |
| 59 | if (Attrs.UnrollCount > 0) { |
| 60 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), |
| 61 | ConstantAsMetadata::get(ConstantInt::get( |
| 62 | Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; |
| 63 | Args.push_back(MDNode::get(Ctx, Vals)); |
| 64 | } |
| 65 | |
| 66 | // Setting vectorize.enable |
| 67 | if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { |
| 68 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), |
| 69 | ConstantAsMetadata::get(ConstantInt::get( |
| 70 | Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == |
| 71 | LoopAttributes::Enable)))}; |
| 72 | Args.push_back(MDNode::get(Ctx, Vals)); |
| 73 | } |
| 74 | |
| 75 | // Setting unroll.full or unroll.disable |
| 76 | if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { |
| 77 | std::string Name; |
| 78 | if (Attrs.UnrollEnable == LoopAttributes::Enable) |
| 79 | Name = "llvm.loop.unroll.enable"; |
| 80 | else if (Attrs.UnrollEnable == LoopAttributes::Full) |
| 81 | Name = "llvm.loop.unroll.full"; |
| 82 | else |
| 83 | Name = "llvm.loop.unroll.disable"; |
| 84 | Metadata *Vals[] = {MDString::get(Ctx, Name)}; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 85 | Args.push_back(MDNode::get(Ctx, Vals)); |
| 86 | } |
| 87 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 88 | if (Attrs.DistributeEnable != LoopAttributes::Unspecified) { |
| 89 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), |
| 90 | ConstantAsMetadata::get(ConstantInt::get( |
| 91 | Type::getInt1Ty(Ctx), (Attrs.DistributeEnable == |
| 92 | LoopAttributes::Enable)))}; |
| 93 | Args.push_back(MDNode::get(Ctx, Vals)); |
| 94 | } |
| 95 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 96 | // Set the first operand to itself. |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 97 | MDNode *LoopID = MDNode::get(Ctx, Args); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 98 | LoopID->replaceOperandWith(0, LoopID); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 99 | return LoopID; |
| 100 | } |
| 101 | |
| 102 | LoopAttributes::LoopAttributes(bool IsParallel) |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 103 | : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), |
| 104 | UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0), |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 105 | InterleaveCount(0), UnrollCount(0), |
| 106 | DistributeEnable(LoopAttributes::Unspecified) {} |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 107 | |
| 108 | void LoopAttributes::clear() { |
| 109 | IsParallel = false; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 110 | VectorizeWidth = 0; |
| 111 | InterleaveCount = 0; |
| 112 | UnrollCount = 0; |
| 113 | VectorizeEnable = LoopAttributes::Unspecified; |
| 114 | UnrollEnable = LoopAttributes::Unspecified; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 115 | } |
| 116 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 117 | LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, |
| 118 | llvm::DebugLoc Location) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 119 | : LoopID(nullptr), Header(Header), Attrs(Attrs) { |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 120 | LoopID = createMetadata(Header->getContext(), Attrs, Location); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 121 | } |
| 122 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 123 | void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) { |
| 124 | Active.push_back(LoopInfo(Header, StagedAttrs, Location)); |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 125 | // Clear the attributes so nested loops do not inherit them. |
| 126 | StagedAttrs.clear(); |
| 127 | } |
| 128 | |
| 129 | void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 130 | ArrayRef<const clang::Attr *> Attrs, |
| 131 | llvm::DebugLoc Location) { |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 132 | |
| 133 | // Identify loop hint attributes from Attrs. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 134 | for (const auto *Attr : Attrs) { |
| 135 | const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 136 | const OpenCLUnrollHintAttr *OpenCLHint = |
| 137 | dyn_cast<OpenCLUnrollHintAttr>(Attr); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 138 | |
| 139 | // Skip non loop hint attributes |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 140 | if (!LH && !OpenCLHint) { |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 141 | continue; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 142 | } |
| 143 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 144 | LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; |
| 145 | LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; |
| 146 | unsigned ValueInt = 1; |
| 147 | // Translate opencl_unroll_hint attribute argument to |
| 148 | // equivalent LoopHintAttr enums. |
| 149 | // OpenCL v2.0 s6.11.5: |
| 150 | // 0 - full unroll (no argument). |
| 151 | // 1 - disable unroll. |
| 152 | // other positive integer n - unroll by n. |
| 153 | if (OpenCLHint) { |
| 154 | ValueInt = OpenCLHint->getUnrollHint(); |
| 155 | if (ValueInt == 0) { |
| 156 | State = LoopHintAttr::Full; |
| 157 | } else if (ValueInt != 1) { |
| 158 | Option = LoopHintAttr::UnrollCount; |
| 159 | State = LoopHintAttr::Numeric; |
| 160 | } |
| 161 | } else if (LH) { |
| 162 | auto *ValueExpr = LH->getValue(); |
| 163 | if (ValueExpr) { |
| 164 | llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); |
| 165 | ValueInt = ValueAPS.getSExtValue(); |
| 166 | } |
| 167 | |
| 168 | Option = LH->getOption(); |
| 169 | State = LH->getState(); |
| 170 | } |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 171 | switch (State) { |
| 172 | case LoopHintAttr::Disable: |
| 173 | switch (Option) { |
| 174 | case LoopHintAttr::Vectorize: |
| 175 | // Disable vectorization by specifying a width of 1. |
| 176 | setVectorizeWidth(1); |
| 177 | break; |
| 178 | case LoopHintAttr::Interleave: |
| 179 | // Disable interleaving by speciyfing a count of 1. |
| 180 | setInterleaveCount(1); |
| 181 | break; |
| 182 | case LoopHintAttr::Unroll: |
| 183 | setUnrollState(LoopAttributes::Disable); |
| 184 | break; |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 185 | case LoopHintAttr::Distribute: |
| 186 | setDistributeState(false); |
| 187 | break; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 188 | case LoopHintAttr::UnrollCount: |
| 189 | case LoopHintAttr::VectorizeWidth: |
| 190 | case LoopHintAttr::InterleaveCount: |
| 191 | llvm_unreachable("Options cannot be disabled."); |
| 192 | break; |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 193 | } |
| 194 | break; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 195 | case LoopHintAttr::Enable: |
| 196 | switch (Option) { |
| 197 | case LoopHintAttr::Vectorize: |
| 198 | case LoopHintAttr::Interleave: |
| 199 | setVectorizeEnable(true); |
| 200 | break; |
| 201 | case LoopHintAttr::Unroll: |
| 202 | setUnrollState(LoopAttributes::Enable); |
| 203 | break; |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 204 | case LoopHintAttr::Distribute: |
| 205 | setDistributeState(true); |
| 206 | break; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 207 | case LoopHintAttr::UnrollCount: |
| 208 | case LoopHintAttr::VectorizeWidth: |
| 209 | case LoopHintAttr::InterleaveCount: |
| 210 | llvm_unreachable("Options cannot enabled."); |
| 211 | break; |
| 212 | } |
| 213 | break; |
| 214 | case LoopHintAttr::AssumeSafety: |
| 215 | switch (Option) { |
| 216 | case LoopHintAttr::Vectorize: |
| 217 | case LoopHintAttr::Interleave: |
| 218 | // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. |
| 219 | setParallel(true); |
| 220 | setVectorizeEnable(true); |
| 221 | break; |
| 222 | case LoopHintAttr::Unroll: |
| 223 | case LoopHintAttr::UnrollCount: |
| 224 | case LoopHintAttr::VectorizeWidth: |
| 225 | case LoopHintAttr::InterleaveCount: |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 226 | case LoopHintAttr::Distribute: |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 227 | llvm_unreachable("Options cannot be used to assume mem safety."); |
| 228 | break; |
| 229 | } |
| 230 | break; |
| 231 | case LoopHintAttr::Full: |
| 232 | switch (Option) { |
| 233 | case LoopHintAttr::Unroll: |
| 234 | setUnrollState(LoopAttributes::Full); |
| 235 | break; |
| 236 | case LoopHintAttr::Vectorize: |
| 237 | case LoopHintAttr::Interleave: |
| 238 | case LoopHintAttr::UnrollCount: |
| 239 | case LoopHintAttr::VectorizeWidth: |
| 240 | case LoopHintAttr::InterleaveCount: |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 241 | case LoopHintAttr::Distribute: |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 242 | llvm_unreachable("Options cannot be used with 'full' hint."); |
| 243 | break; |
| 244 | } |
| 245 | break; |
| 246 | case LoopHintAttr::Numeric: |
| 247 | switch (Option) { |
| 248 | case LoopHintAttr::VectorizeWidth: |
| 249 | setVectorizeWidth(ValueInt); |
| 250 | break; |
| 251 | case LoopHintAttr::InterleaveCount: |
| 252 | setInterleaveCount(ValueInt); |
| 253 | break; |
| 254 | case LoopHintAttr::UnrollCount: |
| 255 | setUnrollCount(ValueInt); |
| 256 | break; |
| 257 | case LoopHintAttr::Unroll: |
| 258 | case LoopHintAttr::Vectorize: |
| 259 | case LoopHintAttr::Interleave: |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 260 | case LoopHintAttr::Distribute: |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 261 | llvm_unreachable("Options cannot be assigned a value."); |
| 262 | break; |
| 263 | } |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 264 | break; |
| 265 | } |
| 266 | } |
| 267 | |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 268 | /// Stage the attributes. |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 269 | push(Header, Location); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 270 | } |
| 271 | |
| 272 | void LoopInfoStack::pop() { |
| 273 | assert(!Active.empty() && "No active loops to pop"); |
| 274 | Active.pop_back(); |
| 275 | } |
| 276 | |
| 277 | void LoopInfoStack::InsertHelper(Instruction *I) const { |
| 278 | if (!hasInfo()) |
| 279 | return; |
| 280 | |
| 281 | const LoopInfo &L = getInfo(); |
| 282 | if (!L.getLoopID()) |
| 283 | return; |
| 284 | |
| 285 | if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) { |
| 286 | for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i) |
| 287 | if (TI->getSuccessor(i) == L.getHeader()) { |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 288 | TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID()); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 289 | break; |
| 290 | } |
| 291 | return; |
| 292 | } |
| 293 | |
| 294 | if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory()) |
| 295 | I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID()); |
| 296 | } |