blob: 8e4a0a2a9b585ec2987015ba5eac60ed624bc178 [file] [log] [blame]
Alexander Musman515ad8c2014-05-22 08:54:05 +00001//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Alexander Musman515ad8c2014-05-22 08:54:05 +00006//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
Tyler Nowicki54c020d2015-07-27 20:10:20 +000010#include "clang/AST/ASTContext.h"
Tyler Nowicki9d268e12015-06-11 23:23:17 +000011#include "clang/AST/Attr.h"
Alexander Musman515ad8c2014-05-22 08:54:05 +000012#include "llvm/IR/BasicBlock.h"
Chandler Carruth4aaaaab2018-10-18 08:16:20 +000013#include "llvm/IR/CFG.h"
Alexander Musman515ad8c2014-05-22 08:54:05 +000014#include "llvm/IR/Constants.h"
15#include "llvm/IR/InstrTypes.h"
16#include "llvm/IR/Instructions.h"
17#include "llvm/IR/Metadata.h"
Tyler Nowicki4e8e9002015-06-08 23:27:35 +000018using namespace clang::CodeGen;
Alexander Musman515ad8c2014-05-22 08:54:05 +000019using namespace llvm;
20
Michael Kruse58e76422019-04-01 17:47:41 +000021MDNode *
22LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23 LLVMContext &Ctx = Header->getContext();
24 SmallVector<Metadata *, 4> NewLoopProperties;
25 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26 NewLoopProperties.push_back(TempNode.get());
27 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
Alexander Musman515ad8c2014-05-22 08:54:05 +000028
Michael Kruse58e76422019-04-01 17:47:41 +000029 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30 LoopID->replaceOperandWith(0, LoopID);
31 return LoopID;
32}
33
34MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35 ArrayRef<Metadata *> LoopProperties,
36 bool &HasUserTransforms) {
37 LLVMContext &Ctx = Header->getContext();
38
39 Optional<bool> Enabled;
40 if (Attrs.PipelineDisabled)
41 Enabled = false;
42 else if (Attrs.PipelineInitiationInterval != 0)
43 Enabled = true;
44
45 if (Enabled != true) {
46 SmallVector<Metadata *, 4> NewLoopProperties;
47 if (Enabled == false) {
48 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49 NewLoopProperties.push_back(
50 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51 ConstantAsMetadata::get(ConstantInt::get(
52 llvm::Type::getInt1Ty(Ctx), 1))}));
53 LoopProperties = NewLoopProperties;
54 }
55 return createLoopPropertiesMetadata(LoopProperties);
56 }
Alexander Musman515ad8c2014-05-22 08:54:05 +000057
Duncan P. N. Exon Smithfb494912014-12-09 18:39:32 +000058 SmallVector<Metadata *, 4> Args;
Michael Kruse58e76422019-04-01 17:47:41 +000059 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
Duncan P. N. Exon Smith7fd74ac2015-01-19 21:30:48 +000060 Args.push_back(TempNode.get());
Michael Kruse58e76422019-04-01 17:47:41 +000061 Args.append(LoopProperties.begin(), LoopProperties.end());
Aaron Ballman9bdf5152019-01-04 17:20:00 +000062
63 if (Attrs.PipelineInitiationInterval > 0) {
64 Metadata *Vals[] = {
65 MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66 ConstantAsMetadata::get(ConstantInt::get(
Michael Kruse58e76422019-04-01 17:47:41 +000067 llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
Aaron Ballman9bdf5152019-01-04 17:20:00 +000068 Args.push_back(MDNode::get(Ctx, Vals));
69 }
70
Michael Kruse58e76422019-04-01 17:47:41 +000071 // No follow-up: This is the last transformation.
72
73 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74 LoopID->replaceOperandWith(0, LoopID);
75 HasUserTransforms = true;
76 return LoopID;
77}
78
79MDNode *
80LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81 ArrayRef<Metadata *> LoopProperties,
82 bool &HasUserTransforms) {
83 LLVMContext &Ctx = Header->getContext();
84
85 Optional<bool> Enabled;
86 if (Attrs.UnrollEnable == LoopAttributes::Disable)
87 Enabled = false;
88 else if (Attrs.UnrollEnable == LoopAttributes::Full)
89 Enabled = None;
90 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91 Attrs.UnrollCount != 0)
92 Enabled = true;
93
94 if (Enabled != true) {
95 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96 // if unrolling is disabled.
97 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98 }
99
100 SmallVector<Metadata *, 4> FollowupLoopProperties;
101
102 // Apply all loop properties to the unrolled loop.
103 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104
105 // Don't unroll an already unrolled loop.
106 FollowupLoopProperties.push_back(
107 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108
109 bool FollowupHasTransforms = false;
110 MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111 FollowupHasTransforms);
112
113 SmallVector<Metadata *, 4> Args;
114 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115 Args.push_back(TempNode.get());
116 Args.append(LoopProperties.begin(), LoopProperties.end());
117
118 // Setting unroll.count
119 if (Attrs.UnrollCount > 0) {
120 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121 ConstantAsMetadata::get(ConstantInt::get(
122 llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123 Args.push_back(MDNode::get(Ctx, Vals));
124 }
125
126 // Setting unroll.full or unroll.disable
127 if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129 Args.push_back(MDNode::get(Ctx, Vals));
130 }
131
132 if (FollowupHasTransforms)
133 Args.push_back(MDNode::get(
134 Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135
136 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137 LoopID->replaceOperandWith(0, LoopID);
138 HasUserTransforms = true;
139 return LoopID;
140}
141
142MDNode *
143LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144 ArrayRef<Metadata *> LoopProperties,
145 bool &HasUserTransforms) {
146 LLVMContext &Ctx = Header->getContext();
147
148 Optional<bool> Enabled;
149 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150 Enabled = false;
151 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152 Attrs.UnrollAndJamCount != 0)
153 Enabled = true;
154
155 if (Enabled != true) {
156 SmallVector<Metadata *, 4> NewLoopProperties;
157 if (Enabled == false) {
158 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159 NewLoopProperties.push_back(MDNode::get(
160 Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161 LoopProperties = NewLoopProperties;
162 }
163 return createPartialUnrollMetadata(Attrs, LoopProperties,
164 HasUserTransforms);
165 }
166
167 SmallVector<Metadata *, 4> FollowupLoopProperties;
168 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169 FollowupLoopProperties.push_back(
170 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171
172 bool FollowupHasTransforms = false;
173 MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174 FollowupHasTransforms);
175
176 SmallVector<Metadata *, 4> Args;
177 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178 Args.push_back(TempNode.get());
179 Args.append(LoopProperties.begin(), LoopProperties.end());
180
181 // Setting unroll_and_jam.count
182 if (Attrs.UnrollAndJamCount > 0) {
183 Metadata *Vals[] = {
184 MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186 Attrs.UnrollAndJamCount))};
187 Args.push_back(MDNode::get(Ctx, Vals));
188 }
189
190 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
191 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192 Args.push_back(MDNode::get(Ctx, Vals));
193 }
194
195 if (FollowupHasTransforms)
196 Args.push_back(MDNode::get(
197 Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198 Followup}));
199
200 if (UnrollAndJamInnerFollowup)
201 Args.push_back(MDNode::get(
202 Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203 UnrollAndJamInnerFollowup}));
204
205 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206 LoopID->replaceOperandWith(0, LoopID);
207 HasUserTransforms = true;
208 return LoopID;
209}
210
211MDNode *
212LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213 ArrayRef<Metadata *> LoopProperties,
214 bool &HasUserTransforms) {
215 LLVMContext &Ctx = Header->getContext();
216
217 Optional<bool> Enabled;
218 if (Attrs.VectorizeEnable == LoopAttributes::Disable)
219 Enabled = false;
220 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000221 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
Michael Kruse58e76422019-04-01 17:47:41 +0000222 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
223 Enabled = true;
224
225 if (Enabled != true) {
226 SmallVector<Metadata *, 4> NewLoopProperties;
227 if (Enabled == false) {
228 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229 NewLoopProperties.push_back(
230 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
231 ConstantAsMetadata::get(ConstantInt::get(
232 llvm::Type::getInt1Ty(Ctx), 0))}));
233 LoopProperties = NewLoopProperties;
234 }
235 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236 }
237
238 // Apply all loop properties to the vectorized loop.
239 SmallVector<Metadata *, 4> FollowupLoopProperties;
240 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
241
242 // Don't vectorize an already vectorized loop.
243 FollowupLoopProperties.push_back(
244 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
245
246 bool FollowupHasTransforms = false;
247 MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248 FollowupHasTransforms);
249
250 SmallVector<Metadata *, 4> Args;
251 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
252 Args.push_back(TempNode.get());
253 Args.append(LoopProperties.begin(), LoopProperties.end());
254
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000255 // Setting vectorize.predicate
256 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
257 Metadata *Vals[] = {
258 MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
259 ConstantAsMetadata::get(ConstantInt::get(
260 llvm::Type::getInt1Ty(Ctx),
261 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable)))};
262 Args.push_back(MDNode::get(Ctx, Vals));
263 }
264
Michael Kruse58e76422019-04-01 17:47:41 +0000265 // Setting vectorize.width
266 if (Attrs.VectorizeWidth > 0) {
267 Metadata *Vals[] = {
268 MDString::get(Ctx, "llvm.loop.vectorize.width"),
269 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
270 Attrs.VectorizeWidth))};
271 Args.push_back(MDNode::get(Ctx, Vals));
272 }
273
274 // Setting interleave.count
275 if (Attrs.InterleaveCount > 0) {
276 Metadata *Vals[] = {
277 MDString::get(Ctx, "llvm.loop.interleave.count"),
278 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
279 Attrs.InterleaveCount))};
280 Args.push_back(MDNode::get(Ctx, Vals));
281 }
282
283 // Setting vectorize.enable
284 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
285 Metadata *Vals[] = {
286 MDString::get(Ctx, "llvm.loop.vectorize.enable"),
287 ConstantAsMetadata::get(ConstantInt::get(
288 llvm::Type::getInt1Ty(Ctx),
289 (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
290 Args.push_back(MDNode::get(Ctx, Vals));
291 }
292
293 if (FollowupHasTransforms)
294 Args.push_back(MDNode::get(
295 Ctx,
296 {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
297
Duncan P. N. Exon Smithfb494912014-12-09 18:39:32 +0000298 MDNode *LoopID = MDNode::get(Ctx, Args);
Alexander Musman515ad8c2014-05-22 08:54:05 +0000299 LoopID->replaceOperandWith(0, LoopID);
Michael Kruse58e76422019-04-01 17:47:41 +0000300 HasUserTransforms = true;
Alexander Musman515ad8c2014-05-22 08:54:05 +0000301 return LoopID;
302}
303
Michael Kruse58e76422019-04-01 17:47:41 +0000304MDNode *
305LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
306 ArrayRef<Metadata *> LoopProperties,
307 bool &HasUserTransforms) {
308 LLVMContext &Ctx = Header->getContext();
309
310 Optional<bool> Enabled;
311 if (Attrs.DistributeEnable == LoopAttributes::Disable)
312 Enabled = false;
313 if (Attrs.DistributeEnable == LoopAttributes::Enable)
314 Enabled = true;
315
316 if (Enabled != true) {
317 SmallVector<Metadata *, 4> NewLoopProperties;
318 if (Enabled == false) {
319 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
320 NewLoopProperties.push_back(
321 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
322 ConstantAsMetadata::get(ConstantInt::get(
323 llvm::Type::getInt1Ty(Ctx), 0))}));
324 LoopProperties = NewLoopProperties;
325 }
326 return createLoopVectorizeMetadata(Attrs, LoopProperties,
327 HasUserTransforms);
328 }
329
330 bool FollowupHasTransforms = false;
331 MDNode *Followup =
332 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
333
334 SmallVector<Metadata *, 4> Args;
335 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
336 Args.push_back(TempNode.get());
337 Args.append(LoopProperties.begin(), LoopProperties.end());
338
339 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
340 ConstantAsMetadata::get(ConstantInt::get(
341 llvm::Type::getInt1Ty(Ctx),
342 (Attrs.DistributeEnable == LoopAttributes::Enable)))};
343 Args.push_back(MDNode::get(Ctx, Vals));
344
345 if (FollowupHasTransforms)
346 Args.push_back(MDNode::get(
347 Ctx,
348 {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
349
350 MDNode *LoopID = MDNode::get(Ctx, Args);
351 LoopID->replaceOperandWith(0, LoopID);
352 HasUserTransforms = true;
353 return LoopID;
354}
355
356MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
357 ArrayRef<Metadata *> LoopProperties,
358 bool &HasUserTransforms) {
359 LLVMContext &Ctx = Header->getContext();
360
361 Optional<bool> Enabled;
362 if (Attrs.UnrollEnable == LoopAttributes::Disable)
363 Enabled = false;
364 else if (Attrs.UnrollEnable == LoopAttributes::Full)
365 Enabled = true;
366
367 if (Enabled != true) {
368 SmallVector<Metadata *, 4> NewLoopProperties;
369 if (Enabled == false) {
370 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
371 NewLoopProperties.push_back(
372 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
373 LoopProperties = NewLoopProperties;
374 }
375 return createLoopDistributeMetadata(Attrs, LoopProperties,
376 HasUserTransforms);
377 }
378
379 SmallVector<Metadata *, 4> Args;
380 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
381 Args.push_back(TempNode.get());
382 Args.append(LoopProperties.begin(), LoopProperties.end());
383 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
384
385 // No follow-up: there is no loop after full unrolling.
386 // TODO: Warn if there are transformations after full unrolling.
387
388 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
389 LoopID->replaceOperandWith(0, LoopID);
390 HasUserTransforms = true;
391 return LoopID;
392}
393
394MDNode *LoopInfo::createMetadata(
395 const LoopAttributes &Attrs,
396 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
397 bool &HasUserTransforms) {
398 SmallVector<Metadata *, 3> LoopProperties;
399
400 // If we have a valid start debug location for the loop, add it.
401 if (StartLoc) {
402 LoopProperties.push_back(StartLoc.getAsMDNode());
403
404 // If we also have a valid end debug location for the loop, add it.
405 if (EndLoc)
406 LoopProperties.push_back(EndLoc.getAsMDNode());
407 }
408
409 assert(!!AccGroup == Attrs.IsParallel &&
410 "There must be an access group iff the loop is parallel");
411 if (Attrs.IsParallel) {
412 LLVMContext &Ctx = Header->getContext();
413 LoopProperties.push_back(MDNode::get(
414 Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
415 }
416
417 LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
418 AdditionalLoopProperties.end());
419 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
420}
421
Alexander Musman515ad8c2014-05-22 08:54:05 +0000422LoopAttributes::LoopAttributes(bool IsParallel)
Tyler Nowickida46d0e2015-07-14 23:03:09 +0000423 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
David Greenc8e39242018-08-01 14:36:12 +0000424 UnrollEnable(LoopAttributes::Unspecified),
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000425 UnrollAndJamEnable(LoopAttributes::Unspecified),
426 VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
David Greenc8e39242018-08-01 14:36:12 +0000427 InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000428 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
429 PipelineInitiationInterval(0) {}
Alexander Musman515ad8c2014-05-22 08:54:05 +0000430
431void LoopAttributes::clear() {
432 IsParallel = false;
Tyler Nowickida46d0e2015-07-14 23:03:09 +0000433 VectorizeWidth = 0;
434 InterleaveCount = 0;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000435 UnrollCount = 0;
David Greenc8e39242018-08-01 14:36:12 +0000436 UnrollAndJamCount = 0;
Tyler Nowickida46d0e2015-07-14 23:03:09 +0000437 VectorizeEnable = LoopAttributes::Unspecified;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000438 UnrollEnable = LoopAttributes::Unspecified;
David Greenc8e39242018-08-01 14:36:12 +0000439 UnrollAndJamEnable = LoopAttributes::Unspecified;
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000440 VectorizePredicateEnable = LoopAttributes::Unspecified;
Adam Nemet9c848592016-08-24 04:31:56 +0000441 DistributeEnable = LoopAttributes::Unspecified;
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000442 PipelineDisabled = false;
443 PipelineInitiationInterval = 0;
Alexander Musman515ad8c2014-05-22 08:54:05 +0000444}
445
Hal Finkelc07e19b2016-05-25 21:53:24 +0000446LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
Michael Kruse58e76422019-04-01 17:47:41 +0000447 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
448 LoopInfo *Parent)
449 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
450 Parent(Parent) {
451
452 if (Attrs.IsParallel) {
453 // Create an access group for this loop.
454 LLVMContext &Ctx = Header->getContext();
455 AccGroup = MDNode::getDistinct(Ctx, {});
456 }
457
458 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
459 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
460 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
461 Attrs.PipelineInitiationInterval == 0 &&
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000462 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
Michael Kruse58e76422019-04-01 17:47:41 +0000463 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
464 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
465 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
466 Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
467 !EndLoc)
468 return;
469
470 TempLoopID = MDNode::getTemporary(Header->getContext(), None);
471}
472
473void LoopInfo::finish() {
474 // We did not annotate the loop body instructions because there are no
475 // attributes for this loop.
476 if (!TempLoopID)
477 return;
478
479 MDNode *LoopID;
480 LoopAttributes CurLoopAttr = Attrs;
481 LLVMContext &Ctx = Header->getContext();
482
483 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
484 Parent->Attrs.UnrollAndJamCount != 0)) {
485 // Parent unroll-and-jams this loop.
486 // Split the transformations in those that happens before the unroll-and-jam
487 // and those after.
488
489 LoopAttributes BeforeJam, AfterJam;
490
491 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
492
493 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
494 BeforeJam.InterleaveCount = Attrs.InterleaveCount;
495 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
496 BeforeJam.DistributeEnable = Attrs.DistributeEnable;
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000497 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
Michael Kruse58e76422019-04-01 17:47:41 +0000498
499 switch (Attrs.UnrollEnable) {
500 case LoopAttributes::Unspecified:
501 case LoopAttributes::Disable:
502 BeforeJam.UnrollEnable = Attrs.UnrollEnable;
503 AfterJam.UnrollEnable = Attrs.UnrollEnable;
504 break;
505 case LoopAttributes::Full:
506 BeforeJam.UnrollEnable = LoopAttributes::Full;
507 break;
508 case LoopAttributes::Enable:
509 AfterJam.UnrollEnable = LoopAttributes::Enable;
510 break;
511 }
512
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000513 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
Michael Kruse58e76422019-04-01 17:47:41 +0000514 AfterJam.UnrollCount = Attrs.UnrollCount;
515 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
516 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
517
518 // If this loop is subject of an unroll-and-jam by the parent loop, and has
519 // an unroll-and-jam annotation itself, we have to decide whether to first
520 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
521 // UnrollAndJam pass processes loops from inner to outer, so we apply the
522 // inner first.
523 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
524 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
525
526 // Set the inner followup metadata to process by the outer loop. Only
527 // consider the first inner loop.
528 if (!Parent->UnrollAndJamInnerFollowup) {
529 // Splitting the attributes into a BeforeJam and an AfterJam part will
530 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
531 // to be forwarded to the AfterJam part. We detect the situation here and
532 // add it manually.
533 SmallVector<Metadata *, 1> BeforeLoopProperties;
534 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000535 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
Michael Kruse58e76422019-04-01 17:47:41 +0000536 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
537 BeforeLoopProperties.push_back(
538 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
539
540 bool InnerFollowupHasTransform = false;
541 MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
542 InnerFollowupHasTransform);
543 if (InnerFollowupHasTransform)
544 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
545 }
546
547 CurLoopAttr = BeforeJam;
548 }
549
550 bool HasUserTransforms = false;
551 LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
552 TempLoopID->replaceAllUsesWith(LoopID);
Alexander Musman515ad8c2014-05-22 08:54:05 +0000553}
554
Benjamin Kramer81cb4b72016-11-24 16:01:20 +0000555void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
556 const llvm::DebugLoc &EndLoc) {
Michael Kruse58e76422019-04-01 17:47:41 +0000557 Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
558 Active.empty() ? nullptr : &Active.back()));
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000559 // Clear the attributes so nested loops do not inherit them.
560 StagedAttrs.clear();
561}
562
563void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
Hal Finkelc07e19b2016-05-25 21:53:24 +0000564 ArrayRef<const clang::Attr *> Attrs,
Benjamin Kramer81cb4b72016-11-24 16:01:20 +0000565 const llvm::DebugLoc &StartLoc,
566 const llvm::DebugLoc &EndLoc) {
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000567
568 // Identify loop hint attributes from Attrs.
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000569 for (const auto *Attr : Attrs) {
570 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000571 const OpenCLUnrollHintAttr *OpenCLHint =
572 dyn_cast<OpenCLUnrollHintAttr>(Attr);
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000573
574 // Skip non loop hint attributes
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000575 if (!LH && !OpenCLHint) {
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000576 continue;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000577 }
578
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000579 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
580 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
581 unsigned ValueInt = 1;
582 // Translate opencl_unroll_hint attribute argument to
583 // equivalent LoopHintAttr enums.
Fangrui Song6907ce22018-07-30 19:24:48 +0000584 // OpenCL v2.0 s6.11.5:
Andrew Savonichev76b178d2019-03-20 16:43:07 +0000585 // 0 - enable unroll (no argument).
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000586 // 1 - disable unroll.
587 // other positive integer n - unroll by n.
588 if (OpenCLHint) {
589 ValueInt = OpenCLHint->getUnrollHint();
590 if (ValueInt == 0) {
Andrew Savonichev76b178d2019-03-20 16:43:07 +0000591 State = LoopHintAttr::Enable;
Anastasia Stulova6bdbcbb2016-02-19 18:30:11 +0000592 } else if (ValueInt != 1) {
593 Option = LoopHintAttr::UnrollCount;
594 State = LoopHintAttr::Numeric;
595 }
596 } else if (LH) {
597 auto *ValueExpr = LH->getValue();
598 if (ValueExpr) {
599 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
600 ValueInt = ValueAPS.getSExtValue();
601 }
602
603 Option = LH->getOption();
604 State = LH->getState();
605 }
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000606 switch (State) {
607 case LoopHintAttr::Disable:
608 switch (Option) {
609 case LoopHintAttr::Vectorize:
610 // Disable vectorization by specifying a width of 1.
611 setVectorizeWidth(1);
612 break;
613 case LoopHintAttr::Interleave:
614 // Disable interleaving by speciyfing a count of 1.
615 setInterleaveCount(1);
616 break;
617 case LoopHintAttr::Unroll:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000618 setUnrollState(LoopAttributes::Disable);
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000619 break;
David Greenc8e39242018-08-01 14:36:12 +0000620 case LoopHintAttr::UnrollAndJam:
621 setUnrollAndJamState(LoopAttributes::Disable);
622 break;
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000623 case LoopHintAttr::VectorizePredicate:
624 setVectorizePredicateState(LoopAttributes::Disable);
625 break;
Adam Nemet2de463e2016-06-14 12:04:26 +0000626 case LoopHintAttr::Distribute:
627 setDistributeState(false);
628 break;
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000629 case LoopHintAttr::PipelineDisabled:
630 setPipelineDisabled(true);
631 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000632 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000633 case LoopHintAttr::UnrollAndJamCount:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000634 case LoopHintAttr::VectorizeWidth:
635 case LoopHintAttr::InterleaveCount:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000636 case LoopHintAttr::PipelineInitiationInterval:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000637 llvm_unreachable("Options cannot be disabled.");
638 break;
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000639 }
640 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000641 case LoopHintAttr::Enable:
642 switch (Option) {
643 case LoopHintAttr::Vectorize:
644 case LoopHintAttr::Interleave:
645 setVectorizeEnable(true);
646 break;
647 case LoopHintAttr::Unroll:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000648 setUnrollState(LoopAttributes::Enable);
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000649 break;
David Greenc8e39242018-08-01 14:36:12 +0000650 case LoopHintAttr::UnrollAndJam:
651 setUnrollAndJamState(LoopAttributes::Enable);
652 break;
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000653 case LoopHintAttr::VectorizePredicate:
654 setVectorizePredicateState(LoopAttributes::Enable);
655 break;
Adam Nemet2de463e2016-06-14 12:04:26 +0000656 case LoopHintAttr::Distribute:
657 setDistributeState(true);
658 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000659 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000660 case LoopHintAttr::UnrollAndJamCount:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000661 case LoopHintAttr::VectorizeWidth:
662 case LoopHintAttr::InterleaveCount:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000663 case LoopHintAttr::PipelineDisabled:
664 case LoopHintAttr::PipelineInitiationInterval:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000665 llvm_unreachable("Options cannot enabled.");
666 break;
667 }
668 break;
669 case LoopHintAttr::AssumeSafety:
670 switch (Option) {
671 case LoopHintAttr::Vectorize:
672 case LoopHintAttr::Interleave:
673 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
674 setParallel(true);
675 setVectorizeEnable(true);
676 break;
677 case LoopHintAttr::Unroll:
David Greenc8e39242018-08-01 14:36:12 +0000678 case LoopHintAttr::UnrollAndJam:
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000679 case LoopHintAttr::VectorizePredicate:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000680 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000681 case LoopHintAttr::UnrollAndJamCount:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000682 case LoopHintAttr::VectorizeWidth:
683 case LoopHintAttr::InterleaveCount:
Adam Nemet2de463e2016-06-14 12:04:26 +0000684 case LoopHintAttr::Distribute:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000685 case LoopHintAttr::PipelineDisabled:
686 case LoopHintAttr::PipelineInitiationInterval:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000687 llvm_unreachable("Options cannot be used to assume mem safety.");
688 break;
689 }
690 break;
Mark Heffernan397a98d2015-08-10 17:29:39 +0000691 case LoopHintAttr::Full:
692 switch (Option) {
693 case LoopHintAttr::Unroll:
694 setUnrollState(LoopAttributes::Full);
695 break;
David Greenc8e39242018-08-01 14:36:12 +0000696 case LoopHintAttr::UnrollAndJam:
697 setUnrollAndJamState(LoopAttributes::Full);
698 break;
Mark Heffernan397a98d2015-08-10 17:29:39 +0000699 case LoopHintAttr::Vectorize:
700 case LoopHintAttr::Interleave:
701 case LoopHintAttr::UnrollCount:
David Greenc8e39242018-08-01 14:36:12 +0000702 case LoopHintAttr::UnrollAndJamCount:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000703 case LoopHintAttr::VectorizeWidth:
704 case LoopHintAttr::InterleaveCount:
Adam Nemet2de463e2016-06-14 12:04:26 +0000705 case LoopHintAttr::Distribute:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000706 case LoopHintAttr::PipelineDisabled:
707 case LoopHintAttr::PipelineInitiationInterval:
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000708 case LoopHintAttr::VectorizePredicate:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000709 llvm_unreachable("Options cannot be used with 'full' hint.");
710 break;
711 }
712 break;
713 case LoopHintAttr::Numeric:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000714 switch (Option) {
715 case LoopHintAttr::VectorizeWidth:
716 setVectorizeWidth(ValueInt);
717 break;
718 case LoopHintAttr::InterleaveCount:
719 setInterleaveCount(ValueInt);
720 break;
721 case LoopHintAttr::UnrollCount:
722 setUnrollCount(ValueInt);
723 break;
David Greenc8e39242018-08-01 14:36:12 +0000724 case LoopHintAttr::UnrollAndJamCount:
725 setUnrollAndJamCount(ValueInt);
726 break;
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000727 case LoopHintAttr::PipelineInitiationInterval:
728 setPipelineInitiationInterval(ValueInt);
729 break;
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000730 case LoopHintAttr::Unroll:
David Greenc8e39242018-08-01 14:36:12 +0000731 case LoopHintAttr::UnrollAndJam:
Sjoerd Meijera48f58c2019-07-25 07:33:13 +0000732 case LoopHintAttr::VectorizePredicate:
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000733 case LoopHintAttr::Vectorize:
734 case LoopHintAttr::Interleave:
Adam Nemet2de463e2016-06-14 12:04:26 +0000735 case LoopHintAttr::Distribute:
Aaron Ballman9bdf5152019-01-04 17:20:00 +0000736 case LoopHintAttr::PipelineDisabled:
Mark Heffernan397a98d2015-08-10 17:29:39 +0000737 llvm_unreachable("Options cannot be assigned a value.");
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000738 break;
739 }
Tyler Nowicki9d268e12015-06-11 23:23:17 +0000740 break;
741 }
742 }
743
Tyler Nowicki54c020d2015-07-27 20:10:20 +0000744 /// Stage the attributes.
Amara Emerson652795d2016-11-10 14:44:30 +0000745 push(Header, StartLoc, EndLoc);
Alexander Musman515ad8c2014-05-22 08:54:05 +0000746}
747
748void LoopInfoStack::pop() {
749 assert(!Active.empty() && "No active loops to pop");
Michael Kruse58e76422019-04-01 17:47:41 +0000750 Active.back().finish();
Alexander Musman515ad8c2014-05-22 08:54:05 +0000751 Active.pop_back();
752}
753
754void LoopInfoStack::InsertHelper(Instruction *I) const {
Michael Kruse05351372018-12-20 21:24:54 +0000755 if (I->mayReadOrWriteMemory()) {
756 SmallVector<Metadata *, 4> AccessGroups;
757 for (const LoopInfo &AL : Active) {
758 // Here we assume that every loop that has an access group is parallel.
759 if (MDNode *Group = AL.getAccessGroup())
760 AccessGroups.push_back(Group);
761 }
762 MDNode *UnionMD = nullptr;
763 if (AccessGroups.size() == 1)
764 UnionMD = cast<MDNode>(AccessGroups[0]);
765 else if (AccessGroups.size() >= 2)
766 UnionMD = MDNode::get(I->getContext(), AccessGroups);
767 I->setMetadata("llvm.access.group", UnionMD);
768 }
769
Alexander Musman515ad8c2014-05-22 08:54:05 +0000770 if (!hasInfo())
771 return;
772
773 const LoopInfo &L = getInfo();
774 if (!L.getLoopID())
775 return;
776
Chandler Carruth4aaaaab2018-10-18 08:16:20 +0000777 if (I->isTerminator()) {
778 for (BasicBlock *Succ : successors(I))
779 if (Succ == L.getHeader()) {
780 I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
Alexander Musman515ad8c2014-05-22 08:54:05 +0000781 break;
782 }
783 return;
784 }
Alexander Musman515ad8c2014-05-22 08:54:05 +0000785}