blob: 3c788fa1dceadb00e7c71490bcf4542a31b512b1 [file] [log] [blame]
Tom Stellarda6f24c62015-12-15 20:55:55 +00001//===-- AMDGPUAnnotateUniformValues.cpp - ---------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This pass adds amdgpu.uniform metadata to IR values so this information
12/// can be used during instruction selection.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPU.h"
17#include "AMDGPUIntrinsicInfo.h"
Alexander Timofeev18009562016-12-08 17:28:47 +000018#include "llvm/ADT/SetVector.h"
Tom Stellarda6f24c62015-12-15 20:55:55 +000019#include "llvm/Analysis/DivergenceAnalysis.h"
Alexander Timofeev18009562016-12-08 17:28:47 +000020#include "llvm/Analysis/LoopInfo.h"
21#include "llvm/Analysis/MemoryDependenceAnalysis.h"
Tom Stellarda6f24c62015-12-15 20:55:55 +000022#include "llvm/IR/IRBuilder.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000023#include "llvm/IR/InstVisitor.h"
Tom Stellarda6f24c62015-12-15 20:55:55 +000024#include "llvm/Support/Debug.h"
25#include "llvm/Support/raw_ostream.h"
26
27#define DEBUG_TYPE "amdgpu-annotate-uniform"
28
29using namespace llvm;
30
31namespace {
32
33class AMDGPUAnnotateUniformValues : public FunctionPass,
34 public InstVisitor<AMDGPUAnnotateUniformValues> {
35 DivergenceAnalysis *DA;
Alexander Timofeev18009562016-12-08 17:28:47 +000036 MemoryDependenceResults *MDR;
37 LoopInfo *LI;
38 DenseMap<Value*, GetElementPtrInst*> noClobberClones;
39 bool isKernelFunc;
Yaxun Liu1a14bfa2017-03-27 14:04:01 +000040 AMDGPUAS AMDGPUASI;
Tom Stellarda6f24c62015-12-15 20:55:55 +000041
42public:
43 static char ID;
44 AMDGPUAnnotateUniformValues() :
45 FunctionPass(ID) { }
46 bool doInitialization(Module &M) override;
47 bool runOnFunction(Function &F) override;
Mehdi Amini117296c2016-10-01 02:56:57 +000048 StringRef getPassName() const override {
49 return "AMDGPU Annotate Uniform Values";
50 }
Tom Stellarda6f24c62015-12-15 20:55:55 +000051 void getAnalysisUsage(AnalysisUsage &AU) const override {
52 AU.addRequired<DivergenceAnalysis>();
Alexander Timofeev18009562016-12-08 17:28:47 +000053 AU.addRequired<MemoryDependenceWrapperPass>();
54 AU.addRequired<LoopInfoWrapperPass>();
Tom Stellarda6f24c62015-12-15 20:55:55 +000055 AU.setPreservesAll();
56 }
57
Tom Stellardbc4497b2016-02-12 23:45:29 +000058 void visitBranchInst(BranchInst &I);
Tom Stellarda6f24c62015-12-15 20:55:55 +000059 void visitLoadInst(LoadInst &I);
Alexander Timofeev18009562016-12-08 17:28:47 +000060 bool isClobberedInFunction(LoadInst * Load);
Tom Stellarda6f24c62015-12-15 20:55:55 +000061};
62
63} // End anonymous namespace
64
65INITIALIZE_PASS_BEGIN(AMDGPUAnnotateUniformValues, DEBUG_TYPE,
66 "Add AMDGPU uniform metadata", false, false)
67INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)
Alexander Timofeev18009562016-12-08 17:28:47 +000068INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
69INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
Tom Stellarda6f24c62015-12-15 20:55:55 +000070INITIALIZE_PASS_END(AMDGPUAnnotateUniformValues, DEBUG_TYPE,
71 "Add AMDGPU uniform metadata", false, false)
72
73char AMDGPUAnnotateUniformValues::ID = 0;
74
Tom Stellardbc4497b2016-02-12 23:45:29 +000075static void setUniformMetadata(Instruction *I) {
76 I->setMetadata("amdgpu.uniform", MDNode::get(I->getContext(), {}));
77}
Alexander Timofeev18009562016-12-08 17:28:47 +000078static void setNoClobberMetadata(Instruction *I) {
79 I->setMetadata("amdgpu.noclobber", MDNode::get(I->getContext(), {}));
80}
81
82static void DFS(BasicBlock *Root, SetVector<BasicBlock*> & Set) {
83 for (auto I : predecessors(Root))
84 if (Set.insert(I))
85 DFS(I, Set);
86}
87
88bool AMDGPUAnnotateUniformValues::isClobberedInFunction(LoadInst * Load) {
89 // 1. get Loop for the Load->getparent();
90 // 2. if it exists, collect all the BBs from the most outer
91 // loop and check for the writes. If NOT - start DFS over all preds.
92 // 3. Start DFS over all preds from the most outer loop header.
93 SetVector<BasicBlock *> Checklist;
94 BasicBlock *Start = Load->getParent();
95 Checklist.insert(Start);
96 const Value *Ptr = Load->getPointerOperand();
97 const Loop *L = LI->getLoopFor(Start);
98 if (L) {
99 const Loop *P = L;
100 do {
101 L = P;
102 P = P->getParentLoop();
103 } while (P);
104 Checklist.insert(L->block_begin(), L->block_end());
105 Start = L->getHeader();
106 }
107
108 DFS(Start, Checklist);
109 for (auto &BB : Checklist) {
110 BasicBlock::iterator StartIt = (BB == Load->getParent()) ?
111 BasicBlock::iterator(Load) : BB->end();
112 if (MDR->getPointerDependencyFrom(MemoryLocation(Ptr),
113 true, StartIt, BB, Load).isClobber())
114 return true;
115 }
116 return false;
117}
Tom Stellardbc4497b2016-02-12 23:45:29 +0000118
119void AMDGPUAnnotateUniformValues::visitBranchInst(BranchInst &I) {
120 if (I.isUnconditional())
121 return;
122
123 Value *Cond = I.getCondition();
124 if (!DA->isUniform(Cond))
125 return;
126
127 setUniformMetadata(I.getParent()->getTerminator());
128}
129
Tom Stellarda6f24c62015-12-15 20:55:55 +0000130void AMDGPUAnnotateUniformValues::visitLoadInst(LoadInst &I) {
131 Value *Ptr = I.getPointerOperand();
132 if (!DA->isUniform(Ptr))
133 return;
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000134 auto isGlobalLoad = [&](LoadInst &Load)->bool {
135 return Load.getPointerAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS;
Alexander Timofeev18009562016-12-08 17:28:47 +0000136 };
137 // We're tracking up to the Function boundaries
138 // We cannot go beyond because of FunctionPass restrictions
139 // Thus we can ensure that memory not clobbered for memory
140 // operations that live in kernel only.
141 bool NotClobbered = isKernelFunc && !isClobberedInFunction(&I);
142 Instruction *PtrI = dyn_cast<Instruction>(Ptr);
143 if (!PtrI && NotClobbered && isGlobalLoad(I)) {
144 if (isa<Argument>(Ptr) || isa<GlobalValue>(Ptr)) {
145 // Lookup for the existing GEP
146 if (noClobberClones.count(Ptr)) {
147 PtrI = noClobberClones[Ptr];
148 } else {
149 // Create GEP of the Value
150 Function *F = I.getParent()->getParent();
151 Value *Idx = Constant::getIntegerValue(
152 Type::getInt32Ty(Ptr->getContext()), APInt(64, 0));
153 // Insert GEP at the entry to make it dominate all uses
154 PtrI = GetElementPtrInst::Create(
155 Ptr->getType()->getPointerElementType(), Ptr,
156 ArrayRef<Value*>(Idx), Twine(""), F->getEntryBlock().getFirstNonPHI());
157 }
158 I.replaceUsesOfWith(Ptr, PtrI);
159 }
160 }
Tom Stellarda6f24c62015-12-15 20:55:55 +0000161
Alexander Timofeev18009562016-12-08 17:28:47 +0000162 if (PtrI) {
Tom Stellardbc4497b2016-02-12 23:45:29 +0000163 setUniformMetadata(PtrI);
Alexander Timofeev18009562016-12-08 17:28:47 +0000164 if (NotClobbered)
165 setNoClobberMetadata(PtrI);
166 }
Tom Stellarda6f24c62015-12-15 20:55:55 +0000167}
168
169bool AMDGPUAnnotateUniformValues::doInitialization(Module &M) {
Yaxun Liu1a14bfa2017-03-27 14:04:01 +0000170 AMDGPUASI = AMDGPU::getAMDGPUAS(M);
Tom Stellarda6f24c62015-12-15 20:55:55 +0000171 return false;
172}
173
174bool AMDGPUAnnotateUniformValues::runOnFunction(Function &F) {
Andrew Kaylor7de74af2016-04-25 22:23:44 +0000175 if (skipFunction(F))
176 return false;
177
Alexander Timofeev18009562016-12-08 17:28:47 +0000178 DA = &getAnalysis<DivergenceAnalysis>();
179 MDR = &getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
180 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
181 isKernelFunc = F.getCallingConv() == CallingConv::AMDGPU_KERNEL;
Tom Stellarda6f24c62015-12-15 20:55:55 +0000182
Alexander Timofeev18009562016-12-08 17:28:47 +0000183 visit(F);
184 noClobberClones.clear();
Tom Stellarda6f24c62015-12-15 20:55:55 +0000185 return true;
186}
187
188FunctionPass *
189llvm::createAMDGPUAnnotateUniformValues() {
190 return new AMDGPUAnnotateUniformValues();
191}