blob: b2bc42bfa0136b4c3069d8884899b4bdbef97e6c [file] [log] [blame]
Alexander Musman515ad8c2014-05-22 08:54:05 +00001//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Alexander Musman515ad8c2014-05-22 08:54:05 +00006//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
Tyler Nowicki54c020d2015-07-27 20:10:20 +000010#include "clang/AST/ASTContext.h"
Tyler Nowicki9d268e12015-06-11 23:23:17 +000011#include "clang/AST/Attr.h"
Alexander Musman515ad8c2014-05-22 08:54:05 +000012#include "llvm/IR/BasicBlock.h"
Chandler Carruth4aaaaab2018-10-18 08:16:20 +000013#include "llvm/IR/CFG.h"
Alexander Musman515ad8c2014-05-22 08:54:05 +000014#include "llvm/IR/Constants.h"
15#include "llvm/IR/InstrTypes.h"
16#include "llvm/IR/Instructions.h"
17#include "llvm/IR/Metadata.h"
Tyler Nowicki4e8e9002015-06-08 23:27:35 +000018using namespace clang::CodeGen;
Alexander Musman515ad8c2014-05-22 08:54:05 +000019using namespace llvm;
20
Michael Kruse58e76422019-04-01 17:47:41 +000021MDNode *
22LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23 LLVMContext &Ctx = Header->getContext();
24 SmallVector<Metadata *, 4> NewLoopProperties;
25 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26 NewLoopProperties.push_back(TempNode.get());
27 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
Alexander Musman515ad8c2014-05-22 08:54:05 +000028
Michael Kruse58e76422019-04-01 17:47:41 +000029 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30 LoopID->replaceOperandWith(0, LoopID);
31 return LoopID;
32}
33
34MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35 ArrayRef<Metadata *> LoopProperties,
36 bool &HasUserTransforms) {
37 LLVMContext &Ctx = Header->getContext();
38
39 Optional<bool> Enabled;
40 if (Attrs.PipelineDisabled)
41 Enabled = false;
42 else if (Attrs.PipelineInitiationInterval != 0)
43 Enabled = true;
44
45 if (Enabled != true) {
46 SmallVector<Metadata *, 4> NewLoopProperties;
47 if (Enabled == false) {
48 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49 NewLoopProperties.push_back(
50 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51 ConstantAsMetadata::get(ConstantInt::get(
52 llvm::Type::getInt1Ty(Ctx), 1))}));
53 LoopProperties = NewLoopProperties;
54 }
55 return createLoopPropertiesMetadata(LoopProperties);
56 }
Alexander Musman515ad8c2014-05-22 08:54:05 +000057
Duncan P. N. Exon Smithfb494912014-12-09 18:39:32 +000058 SmallVector<Metadata *, 4> Args;
Michael Kruse58e76422019-04-01 17:47:41 +000059 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
Duncan P. N. Exon Smith7fd74ac2015-01-19 21:30:48 +000060 Args.push_back(TempNode.get());
Michael Kruse58e76422019-04-01 17:47:41 +000061 Args.append(LoopProperties.begin(), LoopProperties.end());
Aaron Ballman9bdf5152019-01-04 17:20:00 +000062
63 if (Attrs.PipelineInitiationInterval > 0) {
64 Metadata *Vals[] = {
65 MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66 ConstantAsMetadata::get(ConstantInt::get(
Michael Kruse58e76422019-04-01 17:47:41 +000067 llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
Aaron Ballman9bdf5152019-01-04 17:20:00 +000068 Args.push_back(MDNode::get(Ctx, Vals));
69 }
70
Michael Kruse58e76422019-04-01 17:47:41 +000071 // No follow-up: This is the last transformation.
72
73 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74 LoopID->replaceOperandWith(0, LoopID);
75 HasUserTransforms = true;
76 return LoopID;
77}
78
79MDNode *
80LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81 ArrayRef<Metadata *> LoopProperties,
82 bool &HasUserTransforms) {
83 LLVMContext &Ctx = Header->getContext();
84
85 Optional<bool> Enabled;
86 if (Attrs.UnrollEnable == LoopAttributes::Disable)
87 Enabled = false;
88 else if (Attrs.UnrollEnable == LoopAttributes::Full)
89 Enabled = None;
90 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91 Attrs.UnrollCount != 0)
92 Enabled = true;
93
94 if (Enabled != true) {
95 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96 // if unrolling is disabled.
97 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98 }
99
100 SmallVector<Metadata *, 4> FollowupLoopProperties;
101
102 // Apply all loop properties to the unrolled loop.
103 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104
105 // Don't unroll an already unrolled loop.
106 FollowupLoopProperties.push_back(
107 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108
109 bool FollowupHasTransforms = false;
110 MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111 FollowupHasTransforms);
112
113 SmallVector<Metadata *, 4> Args;
114 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115 Args.push_back(TempNode.get());
116 Args.append(LoopProperties.begin(), LoopProperties.end());
117
118 // Setting unroll.count
119 if (Attrs.UnrollCount > 0) {
120 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121 ConstantAsMetadata::get(ConstantInt::get(
122 llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123 Args.push_back(MDNode::get(Ctx, Vals));
124 }
125
126 // Setting unroll.full or unroll.disable
127 if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129 Args.push_back(MDNode::get(Ctx, Vals));
130 }
131
132 if (FollowupHasTransforms)
133 Args.push_back(MDNode::get(
134 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135
136 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137 LoopID->replaceOperandWith(0, LoopID);
138 HasUserTransforms = true;
139 return LoopID;
140}
141
142MDNode *
143LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144 ArrayRef<Metadata *> LoopProperties,
145 bool &HasUserTransforms) {
146 LLVMContext &Ctx = Header->getContext();
147
148 Optional<bool> Enabled;
149 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150 Enabled = false;
151 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152 Attrs.UnrollAndJamCount != 0)
153 Enabled = true;
154
155 if (Enabled != true) {
156 SmallVector<Metadata *, 4> NewLoopProperties;
157 if (Enabled == false) {
158 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159 NewLoopProperties.push_back(MDNode::get(
160 Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161 LoopProperties = NewLoopProperties;
162 }
163 return createPartialUnrollMetadata(Attrs, LoopProperties,
164 HasUserTransforms);
165 }
166
167 SmallVector<Metadata *, 4> FollowupLoopProperties;
168 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169 FollowupLoopProperties.push_back(
170 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171
172 bool FollowupHasTransforms = false;
173 MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174 FollowupHasTransforms);
175
176 SmallVector<Metadata *, 4> Args;
177 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178 Args.push_back(TempNode.get());
179 Args.append(LoopProperties.begin(), LoopProperties.end());
180
181 // Setting unroll_and_jam.count
182 if (Attrs.UnrollAndJamCount > 0) {
183 Metadata *Vals[] = {
184 MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186 Attrs.UnrollAndJamCount))};
187 Args.push_back(MDNode::get(Ctx, Vals));
188 }
189
190 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
191 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192 Args.push_back(MDNode::get(Ctx, Vals));
193 }
194
195 if (FollowupHasTransforms)
196 Args.push_back(MDNode::get(
197 Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198 Followup}));
199
200 if (UnrollAndJamInnerFollowup)
201 Args.push_back(MDNode::get(
202 Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203 UnrollAndJamInnerFollowup}));
204
205 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206 LoopID->replaceOperandWith(0, LoopID);
207 HasUserTransforms = true;
208 return LoopID;
209}
210
211MDNode *
212LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213 ArrayRef<Metadata *> LoopProperties,
214 bool &HasUserTransforms) {
215 LLVMContext &Ctx = Header->getContext();
216
217 Optional<bool> Enabled;
218 if (Attrs.VectorizeEnable == LoopAttributes::Disable)
219 Enabled = false;
220 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
221 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
222 Enabled = true;
223
224 if (Enabled != true) {
225 SmallVector<Metadata *, 4> NewLoopProperties;
226 if (Enabled == false) {
227 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228 NewLoopProperties.push_back(
229 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
230 ConstantAsMetadata::get(ConstantInt::get(
231 llvm::Type::getInt1Ty(Ctx), 0))}));
232 LoopProperties = NewLoopProperties;
233 }
234 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
235 }
236
237 // Apply all loop properties to the vectorized loop.
238 SmallVector<Metadata *, 4> FollowupLoopProperties;
239 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
240
241 // Don't vectorize an already vectorized loop.
242 FollowupLoopProperties.push_back(
243 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
244
245 bool FollowupHasTransforms = false;
246 MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247 FollowupHasTransforms);
248
249 SmallVector<Metadata *, 4> Args;
250 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
251 Args.push_back(TempNode.get());
252 Args.append(LoopProperties.begin(), LoopProperties.end());
253
254 // Setting vectorize.width
255 if (Attrs.VectorizeWidth > 0) {
256 Metadata *Vals[] = {
257 MDString::get(Ctx, "llvm.loop.vectorize.width"),
258 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
259 Attrs.VectorizeWidth))};
260 Args.push_back(MDNode::get(Ctx, Vals));
261 }
262
263 // Setting interleave.count
264 if (Attrs.InterleaveCount > 0) {
265 Metadata *Vals[] = {
266 MDString::get(Ctx, "llvm.loop.interleave.count"),
267 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
268 Attrs.InterleaveCount))};
269 Args.push_back(MDNode::get(Ctx, Vals));
270 }
271
272 // Setting vectorize.enable
273 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
274 Metadata *Vals[] = {
275 MDString::get(Ctx, "llvm.loop.vectorize.enable"),
276 ConstantAsMetadata::get(ConstantInt::get(
277 llvm::Type::getInt1Ty(Ctx),
278 (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
279 Args.push_back(MDNode::get(Ctx, Vals));
280 }
281
282 if (FollowupHasTransforms)
283 Args.push_back(MDNode::get(
284 Ctx,
285 {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
286
Duncan P. N. Exon Smithfb494912014-12-09 18:39:32 +0000287 MDNode *LoopID = MDNode::get(Ctx, Args);
Alexander Musman515ad8c2014-05-22 08:54:05 +0000288 LoopID->replaceOperandWith(0, LoopID);
Michael Kruse58e76422019-04-01 17:47:41 +0000289 HasUserTransforms = true;
Alexander Musman515ad8c2014-05-22 08:54:05 +0000290 return LoopID;
291}
292
Michael Kruse58e76422019-04-01 17:47:41 +0000293MDNode *
294LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
295 ArrayRef<Metadata *> LoopProperties,
296 bool &HasUserTransforms) {
297 LLVMContext &Ctx = Header->getContext();
298
299 Optional<bool> Enabled;
300 if (Attrs.DistributeEnable == LoopAttributes::Disable)
301 Enabled = false;
302 if (Attrs.DistributeEnable == LoopAttributes::Enable)
303 Enabled = true;
304
305 if (Enabled != true) {
306 SmallVector<Metadata *, 4> NewLoopProperties;
307 if (Enabled == false) {
308 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
309 NewLoopProperties.push_back(
310 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
311 ConstantAsMetadata::get(ConstantInt::get(
312 llvm::Type::getInt1Ty(Ctx), 0))}));
313 LoopProperties = NewLoopProperties;
314 }
315 return createLoopVectorizeMetadata(Attrs, LoopProperties,
316 HasUserTransforms);
317 }
318
319 bool FollowupHasTransforms = false;
320 MDNode *Followup =
321 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
322
323 SmallVector<Metadata *, 4> Args;
324 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
325 Args.push_back(TempNode.get());
326 Args.append(LoopProperties.begin(), LoopProperties.end());
327
328 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
329 ConstantAsMetadata::get(ConstantInt::get(
330 llvm::Type::getInt1Ty(Ctx),
331 (Attrs.DistributeEnable == LoopAttributes::Enable)))};
332 Args.push_back(MDNode::get(Ctx, Vals));
333
334 if (FollowupHasTransforms)
335 Args.push_back(MDNode::get(
336 Ctx,
337 {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
338
339 MDNode *LoopID = MDNode::get(Ctx, Args);
340 LoopID->replaceOperandWith(0, LoopID);
341 HasUserTransforms = true;
342 return LoopID;
343}
344
345MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
346 ArrayRef<Metadata *> LoopProperties,
347 bool &HasUserTransforms) {
348 LLVMContext &Ctx = Header->getContext();
349
350 Optional<bool> Enabled;
351 if (Attrs.UnrollEnable == LoopAttributes::Disable)
352 Enabled = false;
353 else if (Attrs.UnrollEnable == LoopAttributes::Full)
354 Enabled = true;
355
356 if (Enabled != true) {
357 SmallVector<Metadata *, 4> NewLoopProperties;
358 if (Enabled == false) {
359 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
360 NewLoopProperties.push_back(
361 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
362 LoopProperties = NewLoopProperties;
363 }
364 return createLoopDistributeMetadata(Attrs, LoopProperties,
365 HasUserTransforms);
366 }
367
368 SmallVector<Metadata *, 4> Args;
369 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
370 Args.push_back(TempNode.get());
371 Args.append(LoopProperties.begin(), LoopProperties.end());
372 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
373
374 // No follow-up: there is no loop after full unrolling.
375 // TODO: Warn if there are transformations after full unrolling.
376
377 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
378 LoopID->replaceOperandWith(0, LoopID);
379 HasUserTransforms = true;
380 return LoopID;
381}
382
383MDNode *LoopInfo::createMetadata(
384 const LoopAttributes &Attrs,
385 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
386 bool &HasUserTransforms) {
387 SmallVector<Metadata *, 3> LoopProperties;
388
389 // If we have a valid start debug location for the loop, add it.
390 if (StartLoc) {
391 LoopProperties.push_back(StartLoc.getAsMDNode());
392
393 // If we also have a valid end debug location for the loop, add it.
394 if (EndLoc)
395 LoopProperties.push_back(EndLoc.getAsMDNode());
396 }
397
398 assert(!!AccGroup == Attrs.IsParallel &&
399 "There must be an access group iff the loop is parallel");
400 if (Attrs.IsParallel) {
401 LLVMContext &Ctx = Header->getContext();
402 LoopProperties.push_back(MDNode::get(
403 Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
404 }
405
406 LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
407 AdditionalLoopProperties.end());
408 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
409}
410
Alexander Musman515ad8c2014-05-22 08:54:05 +0000411LoopAttributes::LoopAttributes(bool IsParallel)
Tyler Nowickida46d0e2015-07-14 23:03:09 +0000412 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
David Greenc8e39242018-08-01 14:36:12 +0000413 UnrollEnable(LoopAttributes::Unspecified),
414 UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
415 InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000416 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
417 PipelineInitiationInterval(0) {}
Alexander Musman515ad8c2014-05-22 08:54:05 +0000418
419void LoopAttributes::clear() {
420 IsParallel = false;
Tyler Nowickida46d0e2015-07-14 23:03:09 +0000421 VectorizeWidth = 0;
422 InterleaveCount = 0;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000423 UnrollCount = 0;
David Greenc8e39242018-08-01 14:36:12 +0000424 UnrollAndJamCount = 0;
Tyler Nowickida46d0e2015-07-14 23:03:09 +0000425 VectorizeEnable = LoopAttributes::Unspecified;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000426 UnrollEnable = LoopAttributes::Unspecified;
David Greenc8e39242018-08-01 14:36:12 +0000427 UnrollAndJamEnable = LoopAttributes::Unspecified;
Adam Nemet9c848592016-08-24 04:31:56 +0000428 DistributeEnable = LoopAttributes::Unspecified;
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000429 PipelineDisabled = false;
430 PipelineInitiationInterval = 0;
Alexander Musman515ad8c2014-05-22 08:54:05 +0000431}
432
Hal Finkelc07e19b2016-05-25 21:53:24 +0000433LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
Michael Kruse58e76422019-04-01 17:47:41 +0000434 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
435 LoopInfo *Parent)
436 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
437 Parent(Parent) {
438
439 if (Attrs.IsParallel) {
440 // Create an access group for this loop.
441 LLVMContext &Ctx = Header->getContext();
442 AccGroup = MDNode::getDistinct(Ctx, {});
443 }
444
445 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
446 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
447 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
448 Attrs.PipelineInitiationInterval == 0 &&
449 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
450 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
451 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
452 Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
453 !EndLoc)
454 return;
455
456 TempLoopID = MDNode::getTemporary(Header->getContext(), None);
457}
458
459void LoopInfo::finish() {
460 // We did not annotate the loop body instructions because there are no
461 // attributes for this loop.
462 if (!TempLoopID)
463 return;
464
465 MDNode *LoopID;
466 LoopAttributes CurLoopAttr = Attrs;
467 LLVMContext &Ctx = Header->getContext();
468
469 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
470 Parent->Attrs.UnrollAndJamCount != 0)) {
471 // Parent unroll-and-jams this loop.
472 // Split the transformations in those that happens before the unroll-and-jam
473 // and those after.
474
475 LoopAttributes BeforeJam, AfterJam;
476
477 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
478
479 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
480 BeforeJam.InterleaveCount = Attrs.InterleaveCount;
481 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
482 BeforeJam.DistributeEnable = Attrs.DistributeEnable;
483
484 switch (Attrs.UnrollEnable) {
485 case LoopAttributes::Unspecified:
486 case LoopAttributes::Disable:
487 BeforeJam.UnrollEnable = Attrs.UnrollEnable;
488 AfterJam.UnrollEnable = Attrs.UnrollEnable;
489 break;
490 case LoopAttributes::Full:
491 BeforeJam.UnrollEnable = LoopAttributes::Full;
492 break;
493 case LoopAttributes::Enable:
494 AfterJam.UnrollEnable = LoopAttributes::Enable;
495 break;
496 }
497
498 AfterJam.UnrollCount = Attrs.UnrollCount;
499 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
500 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
501
502 // If this loop is subject of an unroll-and-jam by the parent loop, and has
503 // an unroll-and-jam annotation itself, we have to decide whether to first
504 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
505 // UnrollAndJam pass processes loops from inner to outer, so we apply the
506 // inner first.
507 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
508 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
509
510 // Set the inner followup metadata to process by the outer loop. Only
511 // consider the first inner loop.
512 if (!Parent->UnrollAndJamInnerFollowup) {
513 // Splitting the attributes into a BeforeJam and an AfterJam part will
514 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
515 // to be forwarded to the AfterJam part. We detect the situation here and
516 // add it manually.
517 SmallVector<Metadata *, 1> BeforeLoopProperties;
518 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
519 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
520 BeforeLoopProperties.push_back(
521 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
522
523 bool InnerFollowupHasTransform = false;
524 MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
525 InnerFollowupHasTransform);
526 if (InnerFollowupHasTransform)
527 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
528 }
529
530 CurLoopAttr = BeforeJam;
531 }
532
533 bool HasUserTransforms = false;
534 LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
535 TempLoopID->replaceAllUsesWith(LoopID);
Alexander Musman515ad8c2014-05-22 08:54:05 +0000536}
537
Benjamin Kramer81cb4b72016-11-24 16:01:20 +0000538void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
539 const llvm::DebugLoc &EndLoc) {
Michael Kruse58e76422019-04-01 17:47:41 +0000540 Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
541 Active.empty() ? nullptr : &Active.back()));
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000542 // Clear the attributes so nested loops do not inherit them.
543 StagedAttrs.clear();
544}
545
546void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
Hal Finkelc07e19b2016-05-25 21:53:24 +0000547 ArrayRef<const clang::Attr *> Attrs,
Benjamin Kramer81cb4b72016-11-24 16:01:20 +0000548 const llvm::DebugLoc &StartLoc,
549 const llvm::DebugLoc &EndLoc) {
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000550
551 // Identify loop hint attributes from Attrs.
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000552 for (const auto *Attr : Attrs) {
553 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000554 const OpenCLUnrollHintAttr *OpenCLHint =
555 dyn_cast<OpenCLUnrollHintAttr>(Attr);
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000556
557 // Skip non loop hint attributes
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000558 if (!LH && !OpenCLHint) {
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000559 continue;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000560 }
561
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000562 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
563 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
564 unsigned ValueInt = 1;
565 // Translate opencl_unroll_hint attribute argument to
566 // equivalent LoopHintAttr enums.
Fangrui Song6907ce22018-07-30 19:24:48 +0000567 // OpenCL v2.0 s6.11.5:
Andrew Savonichev76b178d2019-03-20 16:43:07 +0000568 // 0 - enable unroll (no argument).
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000569 // 1 - disable unroll.
570 // other positive integer n - unroll by n.
571 if (OpenCLHint) {
572 ValueInt = OpenCLHint->getUnrollHint();
573 if (ValueInt == 0) {
Andrew Savonichev76b178d2019-03-20 16:43:07 +0000574 State = LoopHintAttr::Enable;
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000575 } else if (ValueInt != 1) {
576 Option = LoopHintAttr::UnrollCount;
577 State = LoopHintAttr::Numeric;
578 }
579 } else if (LH) {
580 auto *ValueExpr = LH->getValue();
581 if (ValueExpr) {
582 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
583 ValueInt = ValueAPS.getSExtValue();
584 }
585
586 Option = LH->getOption();
587 State = LH->getState();
588 }
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000589 switch (State) {
590 case LoopHintAttr::Disable:
591 switch (Option) {
592 case LoopHintAttr::Vectorize:
593 // Disable vectorization by specifying a width of 1.
594 setVectorizeWidth(1);
595 break;
596 case LoopHintAttr::Interleave:
597 // Disable interleaving by speciyfing a count of 1.
598 setInterleaveCount(1);
599 break;
600 case LoopHintAttr::Unroll:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000601 setUnrollState(LoopAttributes::Disable);
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000602 break;
David Greenc8e39242018-08-01 14:36:12 +0000603 case LoopHintAttr::UnrollAndJam:
604 setUnrollAndJamState(LoopAttributes::Disable);
605 break;
Adam Nemet2de463e2016-06-14 12:04:26 +0000606 case LoopHintAttr::Distribute:
607 setDistributeState(false);
608 break;
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000609 case LoopHintAttr::PipelineDisabled:
610 setPipelineDisabled(true);
611 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000612 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000613 case LoopHintAttr::UnrollAndJamCount:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000614 case LoopHintAttr::VectorizeWidth:
615 case LoopHintAttr::InterleaveCount:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000616 case LoopHintAttr::PipelineInitiationInterval:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000617 llvm_unreachable("Options cannot be disabled.");
618 break;
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000619 }
620 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000621 case LoopHintAttr::Enable:
622 switch (Option) {
623 case LoopHintAttr::Vectorize:
624 case LoopHintAttr::Interleave:
625 setVectorizeEnable(true);
626 break;
627 case LoopHintAttr::Unroll:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000628 setUnrollState(LoopAttributes::Enable);
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000629 break;
David Greenc8e39242018-08-01 14:36:12 +0000630 case LoopHintAttr::UnrollAndJam:
631 setUnrollAndJamState(LoopAttributes::Enable);
632 break;
Adam Nemet2de463e2016-06-14 12:04:26 +0000633 case LoopHintAttr::Distribute:
634 setDistributeState(true);
635 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000636 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000637 case LoopHintAttr::UnrollAndJamCount:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000638 case LoopHintAttr::VectorizeWidth:
639 case LoopHintAttr::InterleaveCount:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000640 case LoopHintAttr::PipelineDisabled:
641 case LoopHintAttr::PipelineInitiationInterval:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000642 llvm_unreachable("Options cannot enabled.");
643 break;
644 }
645 break;
646 case LoopHintAttr::AssumeSafety:
647 switch (Option) {
648 case LoopHintAttr::Vectorize:
649 case LoopHintAttr::Interleave:
650 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
651 setParallel(true);
652 setVectorizeEnable(true);
653 break;
654 case LoopHintAttr::Unroll:
David Greenc8e39242018-08-01 14:36:12 +0000655 case LoopHintAttr::UnrollAndJam:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000656 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000657 case LoopHintAttr::UnrollAndJamCount:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000658 case LoopHintAttr::VectorizeWidth:
659 case LoopHintAttr::InterleaveCount:
Adam Nemet2de463e2016-06-14 12:04:26 +0000660 case LoopHintAttr::Distribute:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000661 case LoopHintAttr::PipelineDisabled:
662 case LoopHintAttr::PipelineInitiationInterval:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000663 llvm_unreachable("Options cannot be used to assume mem safety.");
664 break;
665 }
666 break;
Mark Heffernan397a98d2015-08-10 17:29:39 +0000667 case LoopHintAttr::Full:
668 switch (Option) {
669 case LoopHintAttr::Unroll:
670 setUnrollState(LoopAttributes::Full);
671 break;
David Greenc8e39242018-08-01 14:36:12 +0000672 case LoopHintAttr::UnrollAndJam:
673 setUnrollAndJamState(LoopAttributes::Full);
674 break;
Mark Heffernan397a98d2015-08-10 17:29:39 +0000675 case LoopHintAttr::Vectorize:
676 case LoopHintAttr::Interleave:
677 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000678 case LoopHintAttr::UnrollAndJamCount:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000679 case LoopHintAttr::VectorizeWidth:
680 case LoopHintAttr::InterleaveCount:
Adam Nemet2de463e2016-06-14 12:04:26 +0000681 case LoopHintAttr::Distribute:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000682 case LoopHintAttr::PipelineDisabled:
683 case LoopHintAttr::PipelineInitiationInterval:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000684 llvm_unreachable("Options cannot be used with 'full' hint.");
685 break;
686 }
687 break;
688 case LoopHintAttr::Numeric:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000689 switch (Option) {
690 case LoopHintAttr::VectorizeWidth:
691 setVectorizeWidth(ValueInt);
692 break;
693 case LoopHintAttr::InterleaveCount:
694 setInterleaveCount(ValueInt);
695 break;
696 case LoopHintAttr::UnrollCount:
697 setUnrollCount(ValueInt);
698 break;
David Greenc8e39242018-08-01 14:36:12 +0000699 case LoopHintAttr::UnrollAndJamCount:
700 setUnrollAndJamCount(ValueInt);
701 break;
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000702 case LoopHintAttr::PipelineInitiationInterval:
703 setPipelineInitiationInterval(ValueInt);
704 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000705 case LoopHintAttr::Unroll:
David Greenc8e39242018-08-01 14:36:12 +0000706 case LoopHintAttr::UnrollAndJam:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000707 case LoopHintAttr::Vectorize:
708 case LoopHintAttr::Interleave:
Adam Nemet2de463e2016-06-14 12:04:26 +0000709 case LoopHintAttr::Distribute:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000710 case LoopHintAttr::PipelineDisabled:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000711 llvm_unreachable("Options cannot be assigned a value.");
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000712 break;
713 }
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000714 break;
715 }
716 }
717
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000718 /// Stage the attributes.
Amara Emerson652795d2016-11-10 14:44:30 +0000719 push(Header, StartLoc, EndLoc);
Alexander Musman515ad8c2014-05-22 08:54:05 +0000720}
721
722void LoopInfoStack::pop() {
723 assert(!Active.empty() && "No active loops to pop");
Michael Kruse58e76422019-04-01 17:47:41 +0000724 Active.back().finish();
Alexander Musman515ad8c2014-05-22 08:54:05 +0000725 Active.pop_back();
726}
727
728void LoopInfoStack::InsertHelper(Instruction *I) const {
Michael Kruse05351372018-12-20 21:24:54 +0000729 if (I->mayReadOrWriteMemory()) {
730 SmallVector<Metadata *, 4> AccessGroups;
731 for (const LoopInfo &AL : Active) {
732 // Here we assume that every loop that has an access group is parallel.
733 if (MDNode *Group = AL.getAccessGroup())
734 AccessGroups.push_back(Group);
735 }
736 MDNode *UnionMD = nullptr;
737 if (AccessGroups.size() == 1)
738 UnionMD = cast<MDNode>(AccessGroups[0]);
739 else if (AccessGroups.size() >= 2)
740 UnionMD = MDNode::get(I->getContext(), AccessGroups);
741 I->setMetadata("llvm.access.group", UnionMD);
742 }
743
Alexander Musman515ad8c2014-05-22 08:54:05 +0000744 if (!hasInfo())
745 return;
746
747 const LoopInfo &L = getInfo();
748 if (!L.getLoopID())
749 return;
750
Chandler Carruth4aaaaab2018-10-18 08:16:20 +0000751 if (I->isTerminator()) {
752 for (BasicBlock *Succ : successors(I))
753 if (Succ == L.getHeader()) {
754 I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
Alexander Musman515ad8c2014-05-22 08:54:05 +0000755 break;
756 }
757 return;
758 }
Alexander Musman515ad8c2014-05-22 08:54:05 +0000759}