[CodeGen] Generate follow-up metadata for loops with more than one transformation.
Before this patch, CGLoop would dump all transformations for a loop into
a single LoopID without encoding any order in which to apply them.
rL348944 added the possibility to encode a transformation order using
followup-attributes.
When a loop has more than one transformation, use the follow-up
attribute define the order in which they are applied. The emitted order
is the defacto order as defined by the current LLVM pass pipeline,
which is:
LoopFullUnrollPass
LoopDistributePass
LoopVectorizePass
LoopUnrollAndJamPass
LoopUnrollPass
MachinePipeliner
This patch should therefore not change the assembly output, assuming
that all explicit transformations can be applied, and no implicit
transformations in-between. In the former case,
WarnMissedTransformationsPass should emit a warning (except for
MachinePipeliner which is not implemented yet). The latter could be
avoided by adding 'llvm.loop.disable_nonforced' attributes.
Because LoopUnrollAndJamPass processes a loop nest, generation of the
MDNode is delayed to after the inner loop metadata have been processed.
A temporary LoopID is therefore used to annotate instructions and
RAUW'ed by the actual LoopID later.
Differential Revision: https://reviews.llvm.org/D57978
llvm-svn: 357415
diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp
index 36ae39b..b2bc42bf 100644
--- a/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -18,138 +18,396 @@
using namespace clang::CodeGen;
using namespace llvm;
-static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
- const llvm::DebugLoc &StartLoc,
- const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) {
+MDNode *
+LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
+ LLVMContext &Ctx = Header->getContext();
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ NewLoopProperties.push_back(TempNode.get());
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
- if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
- Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
- Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
- Attrs.PipelineInitiationInterval == 0 &&
- Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
- Attrs.UnrollEnable == LoopAttributes::Unspecified &&
- Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
- Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
- !EndLoc)
- return nullptr;
+ MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
+ LoopID->replaceOperandWith(0, LoopID);
+ return LoopID;
+}
+
+MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.PipelineDisabled)
+ Enabled = false;
+ else if (Attrs.PipelineInitiationInterval != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), 1))}));
+ LoopProperties = NewLoopProperties;
+ }
+ return createLoopPropertiesMetadata(LoopProperties);
+ }
SmallVector<Metadata *, 4> Args;
- // Reserve operand 0 for loop id self reference.
- auto TempNode = MDNode::getTemporary(Ctx, None);
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
Args.push_back(TempNode.get());
-
- // If we have a valid start debug location for the loop, add it.
- if (StartLoc) {
- Args.push_back(StartLoc.getAsMDNode());
-
- // If we also have a valid end debug location for the loop, add it.
- if (EndLoc)
- Args.push_back(EndLoc.getAsMDNode());
- }
-
- // Setting vectorize.width
- if (Attrs.VectorizeWidth > 0) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- // Setting interleave.count
- if (Attrs.InterleaveCount > 0) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- // Setting unroll.count
- if (Attrs.UnrollCount > 0) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- // Setting unroll_and_jam.count
- if (Attrs.UnrollAndJamCount > 0) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- // Setting vectorize.enable
- if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
- LoopAttributes::Enable)))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- // Setting unroll.full or unroll.disable
- if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
- std::string Name;
- if (Attrs.UnrollEnable == LoopAttributes::Enable)
- Name = "llvm.loop.unroll.enable";
- else if (Attrs.UnrollEnable == LoopAttributes::Full)
- Name = "llvm.loop.unroll.full";
- else
- Name = "llvm.loop.unroll.disable";
- Metadata *Vals[] = {MDString::get(Ctx, Name)};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- // Setting unroll_and_jam.full or unroll_and_jam.disable
- if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) {
- std::string Name;
- if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable)
- Name = "llvm.loop.unroll_and_jam.enable";
- else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full)
- Name = "llvm.loop.unroll_and_jam.full";
- else
- Name = "llvm.loop.unroll_and_jam.disable";
- Metadata *Vals[] = {MDString::get(Ctx, Name)};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
- Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
- LoopAttributes::Enable)))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
-
- if (Attrs.IsParallel) {
- AccGroup = MDNode::getDistinct(Ctx, {});
- Args.push_back(MDNode::get(
- Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
- }
-
- if (Attrs.PipelineDisabled) {
- Metadata *Vals[] = {
- MDString::get(Ctx, "llvm.loop.pipeline.disable"),
- ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))};
- Args.push_back(MDNode::get(Ctx, Vals));
- }
+ Args.append(LoopProperties.begin(), LoopProperties.end());
if (Attrs.PipelineInitiationInterval > 0) {
Metadata *Vals[] = {
MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
ConstantAsMetadata::get(ConstantInt::get(
- Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
+ llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
Args.push_back(MDNode::get(Ctx, Vals));
}
- // Set the first operand to itself.
+ // No follow-up: This is the last transformation.
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *
+LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.UnrollEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.UnrollEnable == LoopAttributes::Full)
+ Enabled = None;
+ else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
+ Attrs.UnrollCount != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
+ // if unrolling is disabled.
+ return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
+ }
+
+ SmallVector<Metadata *, 4> FollowupLoopProperties;
+
+ // Apply all loop properties to the unrolled loop.
+ FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Don't unroll an already unrolled loop.
+ FollowupLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
+ FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Setting unroll.count
+ if (Attrs.UnrollCount > 0) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ // Setting unroll.full or unroll.disable
+ if (Attrs.UnrollEnable == LoopAttributes::Enable) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ if (FollowupHasTransforms)
+ Args.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *
+LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
+ Attrs.UnrollAndJamCount != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(MDNode::get(
+ Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
+ LoopProperties = NewLoopProperties;
+ }
+ return createPartialUnrollMetadata(Attrs, LoopProperties,
+ HasUserTransforms);
+ }
+
+ SmallVector<Metadata *, 4> FollowupLoopProperties;
+ FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ FollowupLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
+ FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Setting unroll_and_jam.count
+ if (Attrs.UnrollAndJamCount > 0) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
+ ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
+ Attrs.UnrollAndJamCount))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ if (FollowupHasTransforms)
+ Args.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
+ Followup}));
+
+ if (UnrollAndJamInnerFollowup)
+ Args.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
+ UnrollAndJamInnerFollowup}));
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *
+LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.VectorizeEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
+ Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), 0))}));
+ LoopProperties = NewLoopProperties;
+ }
+ return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
+ }
+
+ // Apply all loop properties to the vectorized loop.
+ SmallVector<Metadata *, 4> FollowupLoopProperties;
+ FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Don't vectorize an already vectorized loop.
+ FollowupLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
+ FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
+ // Setting vectorize.width
+ if (Attrs.VectorizeWidth > 0) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.vectorize.width"),
+ ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
+ Attrs.VectorizeWidth))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ // Setting interleave.count
+ if (Attrs.InterleaveCount > 0) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.interleave.count"),
+ ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
+ Attrs.InterleaveCount))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ // Setting vectorize.enable
+ if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
+ Metadata *Vals[] = {
+ MDString::get(Ctx, "llvm.loop.vectorize.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx),
+ (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+ }
+
+ if (FollowupHasTransforms)
+ Args.push_back(MDNode::get(
+ Ctx,
+ {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
+
MDNode *LoopID = MDNode::get(Ctx, Args);
LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
return LoopID;
}
+MDNode *
+LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.DistributeEnable == LoopAttributes::Disable)
+ Enabled = false;
+ if (Attrs.DistributeEnable == LoopAttributes::Enable)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx), 0))}));
+ LoopProperties = NewLoopProperties;
+ }
+ return createLoopVectorizeMetadata(Attrs, LoopProperties,
+ HasUserTransforms);
+ }
+
+ bool FollowupHasTransforms = false;
+ MDNode *Followup =
+ createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+
+ Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
+ ConstantAsMetadata::get(ConstantInt::get(
+ llvm::Type::getInt1Ty(Ctx),
+ (Attrs.DistributeEnable == LoopAttributes::Enable)))};
+ Args.push_back(MDNode::get(Ctx, Vals));
+
+ if (FollowupHasTransforms)
+ Args.push_back(MDNode::get(
+ Ctx,
+ {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
+
+ MDNode *LoopID = MDNode::get(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
+ ArrayRef<Metadata *> LoopProperties,
+ bool &HasUserTransforms) {
+ LLVMContext &Ctx = Header->getContext();
+
+ Optional<bool> Enabled;
+ if (Attrs.UnrollEnable == LoopAttributes::Disable)
+ Enabled = false;
+ else if (Attrs.UnrollEnable == LoopAttributes::Full)
+ Enabled = true;
+
+ if (Enabled != true) {
+ SmallVector<Metadata *, 4> NewLoopProperties;
+ if (Enabled == false) {
+ NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
+ NewLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
+ LoopProperties = NewLoopProperties;
+ }
+ return createLoopDistributeMetadata(Attrs, LoopProperties,
+ HasUserTransforms);
+ }
+
+ SmallVector<Metadata *, 4> Args;
+ TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
+ Args.push_back(TempNode.get());
+ Args.append(LoopProperties.begin(), LoopProperties.end());
+ Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
+
+ // No follow-up: there is no loop after full unrolling.
+ // TODO: Warn if there are transformations after full unrolling.
+
+ MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
+ LoopID->replaceOperandWith(0, LoopID);
+ HasUserTransforms = true;
+ return LoopID;
+}
+
+MDNode *LoopInfo::createMetadata(
+ const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
+ bool &HasUserTransforms) {
+ SmallVector<Metadata *, 3> LoopProperties;
+
+ // If we have a valid start debug location for the loop, add it.
+ if (StartLoc) {
+ LoopProperties.push_back(StartLoc.getAsMDNode());
+
+ // If we also have a valid end debug location for the loop, add it.
+ if (EndLoc)
+ LoopProperties.push_back(EndLoc.getAsMDNode());
+ }
+
+ assert(!!AccGroup == Attrs.IsParallel &&
+ "There must be an access group iff the loop is parallel");
+ if (Attrs.IsParallel) {
+ LLVMContext &Ctx = Header->getContext();
+ LoopProperties.push_back(MDNode::get(
+ Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
+ }
+
+ LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
+ AdditionalLoopProperties.end());
+ return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
+}
+
LoopAttributes::LoopAttributes(bool IsParallel)
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
UnrollEnable(LoopAttributes::Unspecified),
@@ -173,15 +431,114 @@
}
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
- const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
- : LoopID(nullptr), Header(Header), Attrs(Attrs) {
- LoopID =
- createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup);
+ const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
+ LoopInfo *Parent)
+ : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
+ Parent(Parent) {
+
+ if (Attrs.IsParallel) {
+ // Create an access group for this loop.
+ LLVMContext &Ctx = Header->getContext();
+ AccGroup = MDNode::getDistinct(Ctx, {});
+ }
+
+ if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
+ Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
+ Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
+ Attrs.PipelineInitiationInterval == 0 &&
+ Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
+ Attrs.UnrollEnable == LoopAttributes::Unspecified &&
+ Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
+ Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
+ !EndLoc)
+ return;
+
+ TempLoopID = MDNode::getTemporary(Header->getContext(), None);
+}
+
+void LoopInfo::finish() {
+ // We did not annotate the loop body instructions because there are no
+ // attributes for this loop.
+ if (!TempLoopID)
+ return;
+
+ MDNode *LoopID;
+ LoopAttributes CurLoopAttr = Attrs;
+ LLVMContext &Ctx = Header->getContext();
+
+ if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
+ Parent->Attrs.UnrollAndJamCount != 0)) {
+ // Parent unroll-and-jams this loop.
+ // Split the transformations in those that happens before the unroll-and-jam
+ // and those after.
+
+ LoopAttributes BeforeJam, AfterJam;
+
+ BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
+
+ BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
+ BeforeJam.InterleaveCount = Attrs.InterleaveCount;
+ BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
+ BeforeJam.DistributeEnable = Attrs.DistributeEnable;
+
+ switch (Attrs.UnrollEnable) {
+ case LoopAttributes::Unspecified:
+ case LoopAttributes::Disable:
+ BeforeJam.UnrollEnable = Attrs.UnrollEnable;
+ AfterJam.UnrollEnable = Attrs.UnrollEnable;
+ break;
+ case LoopAttributes::Full:
+ BeforeJam.UnrollEnable = LoopAttributes::Full;
+ break;
+ case LoopAttributes::Enable:
+ AfterJam.UnrollEnable = LoopAttributes::Enable;
+ break;
+ }
+
+ AfterJam.UnrollCount = Attrs.UnrollCount;
+ AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
+ AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
+
+ // If this loop is subject of an unroll-and-jam by the parent loop, and has
+ // an unroll-and-jam annotation itself, we have to decide whether to first
+ // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
+ // UnrollAndJam pass processes loops from inner to outer, so we apply the
+ // inner first.
+ BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
+ BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
+
+ // Set the inner followup metadata to process by the outer loop. Only
+ // consider the first inner loop.
+ if (!Parent->UnrollAndJamInnerFollowup) {
+ // Splitting the attributes into a BeforeJam and an AfterJam part will
+ // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
+ // to be forwarded to the AfterJam part. We detect the situation here and
+ // add it manually.
+ SmallVector<Metadata *, 1> BeforeLoopProperties;
+ if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
+ BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
+ BeforeLoopProperties.push_back(
+ MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
+
+ bool InnerFollowupHasTransform = false;
+ MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
+ InnerFollowupHasTransform);
+ if (InnerFollowupHasTransform)
+ Parent->UnrollAndJamInnerFollowup = InnerFollowup;
+ }
+
+ CurLoopAttr = BeforeJam;
+ }
+
+ bool HasUserTransforms = false;
+ LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
+ TempLoopID->replaceAllUsesWith(LoopID);
}
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc) {
- Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc));
+ Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
+ Active.empty() ? nullptr : &Active.back()));
// Clear the attributes so nested loops do not inherit them.
StagedAttrs.clear();
}
@@ -364,6 +721,7 @@
void LoopInfoStack::pop() {
assert(!Active.empty() && "No active loops to pop");
+ Active.back().finish();
Active.pop_back();
}
diff --git a/clang/lib/CodeGen/CGLoopInfo.h b/clang/lib/CodeGen/CGLoopInfo.h
index 115685e..35d0e00 100644
--- a/clang/lib/CodeGen/CGLoopInfo.h
+++ b/clang/lib/CodeGen/CGLoopInfo.h
@@ -78,10 +78,11 @@
public:
/// Construct a new LoopInfo for the loop with entry Header.
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs,
- const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc);
+ const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
+ LoopInfo *Parent);
/// Get the loop id metadata for this loop.
- llvm::MDNode *getLoopID() const { return LoopID; }
+ llvm::MDNode *getLoopID() const { return TempLoopID.get(); }
/// Get the header block of this loop.
llvm::BasicBlock *getHeader() const { return Header; }
@@ -92,15 +93,92 @@
/// Return this loop's access group or nullptr if it does not have one.
llvm::MDNode *getAccessGroup() const { return AccGroup; }
+ /// Create the loop's metadata. Must be called after its nested loops have
+ /// been processed.
+ void finish();
+
private:
/// Loop ID metadata.
- llvm::MDNode *LoopID;
+ llvm::TempMDTuple TempLoopID;
/// Header block of this loop.
llvm::BasicBlock *Header;
/// The attributes for this loop.
LoopAttributes Attrs;
/// The access group for memory accesses parallel to this loop.
llvm::MDNode *AccGroup = nullptr;
+ /// Start location of this loop.
+ llvm::DebugLoc StartLoc;
+ /// End location of this loop.
+ llvm::DebugLoc EndLoc;
+ /// The next outer loop, or nullptr if this is the outermost loop.
+ LoopInfo *Parent;
+ /// If this loop has unroll-and-jam metadata, this can be set by the inner
+ /// loop's LoopInfo to set the llvm.loop.unroll_and_jam.followup_inner
+ /// metadata.
+ llvm::MDNode *UnrollAndJamInnerFollowup = nullptr;
+
+ /// Create a LoopID without any transformations.
+ llvm::MDNode *
+ createLoopPropertiesMetadata(llvm::ArrayRef<llvm::Metadata *> LoopProperties);
+
+ /// Create a LoopID for transformations.
+ ///
+ /// The methods call each other in case multiple transformations are applied
+ /// to a loop. The transformation first to be applied will use LoopID of the
+ /// next transformation in its followup attribute.
+ ///
+ /// @param Attrs The loop's transformations.
+ /// @param LoopProperties Non-transformation properties such as debug
+ /// location, parallel accesses and disabled
+ /// transformations. These are added to the returned
+ /// LoopID.
+ /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
+ /// at least one transformation.
+ ///
+ /// @return A LoopID (metadata node) that can be used for the llvm.loop
+ /// annotation or followup-attribute.
+ /// @{
+ llvm::MDNode *
+ createPipeliningMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createPartialUnrollMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createUnrollAndJamMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createLoopVectorizeMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createLoopDistributeMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ llvm::MDNode *
+ createFullUnrollMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
+ /// @}
+
+ /// Create a LoopID for this loop, including transformation-unspecific
+ /// metadata such as debug location.
+ ///
+ /// @param Attrs This loop's attributes and transformations.
+ /// @param LoopProperties Additional non-transformation properties to add
+ /// to the LoopID, such as transformation-specific
+ /// metadata that are not covered by @p Attrs.
+ /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
+ /// at least one transformation.
+ ///
+ /// @return A LoopID (metadata node) that can be used for the llvm.loop
+ /// annotation.
+ llvm::MDNode *createMetadata(const LoopAttributes &Attrs,
+ llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+ bool &HasUserTransforms);
};
/// A stack of loop information corresponding to loop nesting levels.