blob: 132dd0c8081b7fdc1f24c0d07792f4d737e62c72 [file] [log] [blame]
Andrew Lenharth701f5ac2005-11-28 00:58:09 +00001//===- RSProfiling.cpp - Various profiling using random sampling ----------===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Andrew Lenharth701f5ac2005-11-28 00:58:09 +00007//
8//===----------------------------------------------------------------------===//
9//
10// These passes implement a random sampling based profiling. Different methods
11// of choosing when to sample are supported, as well as different types of
12// profiling. This is done as two passes. The first is a sequence of profiling
Andrew Lenharthbb227c12005-11-28 18:00:38 +000013// passes which insert profiling into the program, and remember what they
14// inserted.
15//
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000016// The second stage duplicates all instructions in a function, ignoring the
17// profiling code, then connects the two versions togeather at the entry and at
18// backedges. At each connection point a choice is made as to whether to jump
19// to the profiled code (take a sample) or execute the unprofiled code.
20//
Gordon Henriksen55cbec32007-10-26 03:03:51 +000021// It is highly recommended that after this pass one runs mem2reg and adce
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000022// (instcombine load-vn gdce dse also are good to run afterwards)
23//
24// This design is intended to make the profiling passes independent of the RS
25// framework, but any profiling pass that implements the RSProfiling interface
26// is compatible with the rs framework (and thus can be sampled)
27//
28// TODO: obviously the block and function profiling are almost identical to the
29// existing ones, so they can be unified (esp since these passes are valid
30// without the rs framework).
31// TODO: Fix choice code so that frequency is not hard coded
32//
33//===----------------------------------------------------------------------===//
34
35#include "llvm/Pass.h"
Owen Anderson50895512009-07-06 18:42:36 +000036#include "llvm/LLVMContext.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000037#include "llvm/Module.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000038#include "llvm/Instructions.h"
39#include "llvm/Constants.h"
40#include "llvm/DerivedTypes.h"
Duncan Sandse2c43042008-04-07 13:45:04 +000041#include "llvm/Intrinsics.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000042#include "llvm/Transforms/Scalar.h"
43#include "llvm/Transforms/Utils/BasicBlockUtils.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000044#include "llvm/Support/CommandLine.h"
Reid Spencer9133fe22007-02-05 23:32:05 +000045#include "llvm/Support/Compiler.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000046#include "llvm/Support/Debug.h"
Torok Edwinc25e7582009-07-11 20:10:48 +000047#include "llvm/Support/ErrorHandling.h"
Daniel Dunbarce63ffb2009-07-25 00:23:56 +000048#include "llvm/Support/raw_ostream.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000049#include "llvm/Transforms/Instrumentation.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000050#include "RSProfiling.h"
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000051#include <set>
52#include <map>
53#include <queue>
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000054using namespace llvm;
55
56namespace {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000057 enum RandomMeth {
58 GBV, GBVO, HOSTCC
59 };
Dan Gohman844731a2008-05-13 00:00:25 +000060}
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000061
Dan Gohman844731a2008-05-13 00:00:25 +000062static cl::opt<RandomMeth> RandomMethod("profile-randomness",
63 cl::desc("How to randomly choose to profile:"),
64 cl::values(
65 clEnumValN(GBV, "global", "global counter"),
66 clEnumValN(GBVO, "ra_global",
67 "register allocated global counter"),
68 clEnumValN(HOSTCC, "rdcc", "cycle counter"),
69 clEnumValEnd));
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000070
Dan Gohman844731a2008-05-13 00:00:25 +000071namespace {
Andrew Lenharth8dc2d502005-11-28 18:10:59 +000072 /// NullProfilerRS - The basic profiler that does nothing. It is the default
73 /// profiler and thus terminates RSProfiler chains. It is useful for
74 /// measuring framework overhead
Nick Lewycky6726b6d2009-10-25 06:33:48 +000075 class NullProfilerRS : public RSProfilers {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000076 public:
Nick Lewyckyecd94c82007-05-06 13:37:16 +000077 static char ID; // Pass identification, replacement for typeid
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000078 bool isProfiling(Value* v) {
79 return false;
80 }
81 bool runOnModule(Module &M) {
82 return false;
83 }
84 void getAnalysisUsage(AnalysisUsage &AU) const {
85 AU.setPreservesAll();
86 }
87 };
Dan Gohman844731a2008-05-13 00:00:25 +000088}
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000089
Dan Gohman844731a2008-05-13 00:00:25 +000090static RegisterAnalysisGroup<RSProfilers> A("Profiling passes");
91static RegisterPass<NullProfilerRS> NP("insert-null-profiling-rs",
92 "Measure profiling framework overhead");
93static RegisterAnalysisGroup<RSProfilers, true> NPT(NP);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000094
Dan Gohman844731a2008-05-13 00:00:25 +000095namespace {
Andrew Lenharth8dc2d502005-11-28 18:10:59 +000096 /// Chooser - Something that chooses when to make a sample of the profiled code
Nick Lewycky6726b6d2009-10-25 06:33:48 +000097 class Chooser {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +000098 public:
Andrew Lenharth8dc2d502005-11-28 18:10:59 +000099 /// ProcessChoicePoint - is called for each basic block inserted to choose
100 /// between normal and sample code
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000101 virtual void ProcessChoicePoint(BasicBlock*) = 0;
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000102 /// PrepFunction - is called once per function before other work is done.
103 /// This gives the opertunity to insert new allocas and such.
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000104 virtual void PrepFunction(Function*) = 0;
105 virtual ~Chooser() {}
106 };
107
108 //Things that implement sampling policies
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000109 //A global value that is read-mod-stored to choose when to sample.
110 //A sample is taken when the global counter hits 0
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000111 class GlobalRandomCounter : public Chooser {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000112 GlobalVariable* Counter;
113 Value* ResetValue;
Dan Gohman6de29f82009-06-15 22:12:54 +0000114 const IntegerType* T;
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000115 public:
Dan Gohman6de29f82009-06-15 22:12:54 +0000116 GlobalRandomCounter(Module& M, const IntegerType* t, uint64_t resetval);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000117 virtual ~GlobalRandomCounter();
118 virtual void PrepFunction(Function* F);
119 virtual void ProcessChoicePoint(BasicBlock* bb);
120 };
121
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000122 //Same is GRC, but allow register allocation of the global counter
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000123 class GlobalRandomCounterOpt : public Chooser {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000124 GlobalVariable* Counter;
125 Value* ResetValue;
126 AllocaInst* AI;
Dan Gohman6de29f82009-06-15 22:12:54 +0000127 const IntegerType* T;
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000128 public:
Dan Gohman6de29f82009-06-15 22:12:54 +0000129 GlobalRandomCounterOpt(Module& M, const IntegerType* t, uint64_t resetval);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000130 virtual ~GlobalRandomCounterOpt();
131 virtual void PrepFunction(Function* F);
132 virtual void ProcessChoicePoint(BasicBlock* bb);
133 };
134
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000135 //Use the cycle counter intrinsic as a source of pseudo randomness when
136 //deciding when to sample.
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000137 class CycleCounter : public Chooser {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000138 uint64_t rm;
Chris Lattnerfebe5f12007-01-07 07:22:20 +0000139 Constant *F;
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000140 public:
141 CycleCounter(Module& m, uint64_t resetmask);
142 virtual ~CycleCounter();
143 virtual void PrepFunction(Function* F);
144 virtual void ProcessChoicePoint(BasicBlock* bb);
145 };
146
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000147 /// ProfilerRS - Insert the random sampling framework
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000148 struct ProfilerRS : public FunctionPass {
Nick Lewyckyecd94c82007-05-06 13:37:16 +0000149 static char ID; // Pass identification, replacement for typeid
Dan Gohmanae73dc12008-09-04 17:05:41 +0000150 ProfilerRS() : FunctionPass(&ID) {}
Devang Patel794fd752007-05-01 21:15:47 +0000151
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000152 std::map<Value*, Value*> TransCache;
153 std::set<BasicBlock*> ChoicePoints;
154 Chooser* c;
155
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000156 //Translate and duplicate values for the new profile free version of stuff
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000157 Value* Translate(Value* v);
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000158 //Duplicate an entire function (with out profiling)
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000159 void Duplicate(Function& F, RSProfilers& LI);
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000160 //Called once for each backedge, handle the insertion of choice points and
161 //the interconection of the two versions of the code
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000162 void ProcessBackEdge(BasicBlock* src, BasicBlock* dst, Function& F);
163 bool runOnFunction(Function& F);
164 bool doInitialization(Module &M);
165 virtual void getAnalysisUsage(AnalysisUsage &AU) const;
166 };
Chris Lattnerd74ea2b2006-05-24 17:04:05 +0000167}
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000168
Dan Gohman844731a2008-05-13 00:00:25 +0000169static RegisterPass<ProfilerRS>
170X("insert-rs-profiling-framework",
171 "Insert random sampling instrumentation framework");
172
Devang Patel19974732007-05-03 01:11:54 +0000173char RSProfilers::ID = 0;
174char NullProfilerRS::ID = 0;
175char ProfilerRS::ID = 0;
Lauro Ramos Venancioc7182882007-05-02 20:37:47 +0000176
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000177//Local utilities
178static void ReplacePhiPred(BasicBlock* btarget,
179 BasicBlock* bold, BasicBlock* bnew);
180
181static void CollapsePhi(BasicBlock* btarget, BasicBlock* bsrc);
182
183template<class T>
184static void recBackEdge(BasicBlock* bb, T& BackEdges,
185 std::map<BasicBlock*, int>& color,
186 std::map<BasicBlock*, int>& depth,
187 std::map<BasicBlock*, int>& finish,
188 int& time);
189
190//find the back edges and where they go to
191template<class T>
192static void getBackEdges(Function& F, T& BackEdges);
193
194
195///////////////////////////////////////
196// Methods of choosing when to profile
197///////////////////////////////////////
198
Dan Gohman6de29f82009-06-15 22:12:54 +0000199GlobalRandomCounter::GlobalRandomCounter(Module& M, const IntegerType* t,
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000200 uint64_t resetval) : T(t) {
Owen Andersoneed707b2009-07-24 23:12:02 +0000201 ConstantInt* Init = ConstantInt::get(T, resetval);
Reid Spencerb83eb642006-10-20 07:07:24 +0000202 ResetValue = Init;
Owen Andersone9b11b42009-07-08 19:03:57 +0000203 Counter = new GlobalVariable(M, T, false, GlobalValue::InternalLinkage,
204 Init, "RandomSteeringCounter");
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000205}
206
207GlobalRandomCounter::~GlobalRandomCounter() {}
208
209void GlobalRandomCounter::PrepFunction(Function* F) {}
210
211void GlobalRandomCounter::ProcessChoicePoint(BasicBlock* bb) {
212 BranchInst* t = cast<BranchInst>(bb->getTerminator());
213
214 //decrement counter
215 LoadInst* l = new LoadInst(Counter, "counter", t);
216
Owen Anderson333c4002009-07-09 23:48:35 +0000217 ICmpInst* s = new ICmpInst(t, ICmpInst::ICMP_EQ, l,
Owen Andersoneed707b2009-07-24 23:12:02 +0000218 ConstantInt::get(T, 0),
Owen Anderson333c4002009-07-09 23:48:35 +0000219 "countercc");
Reid Spencere4d87aa2006-12-23 06:05:41 +0000220
Owen Andersoneed707b2009-07-24 23:12:02 +0000221 Value* nv = BinaryOperator::CreateSub(l, ConstantInt::get(T, 1),
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000222 "counternew", t);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000223 new StoreInst(nv, Counter, t);
224 t->setCondition(s);
225
226 //reset counter
227 BasicBlock* oldnext = t->getSuccessor(0);
Owen Anderson1d0be152009-08-13 21:58:54 +0000228 BasicBlock* resetblock = BasicBlock::Create(bb->getContext(),
229 "reset", oldnext->getParent(),
Gabor Greif051a9502008-04-06 20:25:17 +0000230 oldnext);
231 TerminatorInst* t2 = BranchInst::Create(oldnext, resetblock);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000232 t->setSuccessor(0, resetblock);
233 new StoreInst(ResetValue, Counter, t2);
234 ReplacePhiPred(oldnext, bb, resetblock);
235}
236
Dan Gohman6de29f82009-06-15 22:12:54 +0000237GlobalRandomCounterOpt::GlobalRandomCounterOpt(Module& M, const IntegerType* t,
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000238 uint64_t resetval)
239 : AI(0), T(t) {
Owen Andersoneed707b2009-07-24 23:12:02 +0000240 ConstantInt* Init = ConstantInt::get(T, resetval);
Reid Spencerb83eb642006-10-20 07:07:24 +0000241 ResetValue = Init;
Owen Andersone9b11b42009-07-08 19:03:57 +0000242 Counter = new GlobalVariable(M, T, false, GlobalValue::InternalLinkage,
243 Init, "RandomSteeringCounter");
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000244}
245
246GlobalRandomCounterOpt::~GlobalRandomCounterOpt() {}
247
248void GlobalRandomCounterOpt::PrepFunction(Function* F) {
249 //make a local temporary to cache the global
250 BasicBlock& bb = F->getEntryBlock();
Chris Lattnera0e1b0e2007-04-17 17:51:03 +0000251 BasicBlock::iterator InsertPt = bb.begin();
Owen Anderson50dead02009-07-15 23:53:25 +0000252 AI = new AllocaInst(T, 0, "localcounter", InsertPt);
Chris Lattnera0e1b0e2007-04-17 17:51:03 +0000253 LoadInst* l = new LoadInst(Counter, "counterload", InsertPt);
254 new StoreInst(l, AI, InsertPt);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000255
Andrew Lenharth8dc2d502005-11-28 18:10:59 +0000256 //modify all functions and return values to restore the local variable to/from
257 //the global variable
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000258 for(Function::iterator fib = F->begin(), fie = F->end();
259 fib != fie; ++fib)
260 for(BasicBlock::iterator bib = fib->begin(), bie = fib->end();
261 bib != bie; ++bib)
Chris Lattnera0e1b0e2007-04-17 17:51:03 +0000262 if (isa<CallInst>(bib)) {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000263 LoadInst* l = new LoadInst(AI, "counter", bib);
264 new StoreInst(l, Counter, bib);
Chris Lattnera0e1b0e2007-04-17 17:51:03 +0000265 l = new LoadInst(Counter, "counter", ++bib);
266 new StoreInst(l, AI, bib--);
267 } else if (isa<InvokeInst>(bib)) {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000268 LoadInst* l = new LoadInst(AI, "counter", bib);
269 new StoreInst(l, Counter, bib);
270
Chris Lattnera0e1b0e2007-04-17 17:51:03 +0000271 BasicBlock* bb = cast<InvokeInst>(bib)->getNormalDest();
Dan Gohman02dea8b2008-05-23 21:05:58 +0000272 BasicBlock::iterator i = bb->getFirstNonPHI();
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000273 l = new LoadInst(Counter, "counter", i);
274
Chris Lattnera0e1b0e2007-04-17 17:51:03 +0000275 bb = cast<InvokeInst>(bib)->getUnwindDest();
Dan Gohman02dea8b2008-05-23 21:05:58 +0000276 i = bb->getFirstNonPHI();
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000277 l = new LoadInst(Counter, "counter", i);
Chris Lattnera0e1b0e2007-04-17 17:51:03 +0000278 new StoreInst(l, AI, i);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000279 } else if (isa<UnwindInst>(&*bib) || isa<ReturnInst>(&*bib)) {
280 LoadInst* l = new LoadInst(AI, "counter", bib);
281 new StoreInst(l, Counter, bib);
282 }
283}
284
285void GlobalRandomCounterOpt::ProcessChoicePoint(BasicBlock* bb) {
286 BranchInst* t = cast<BranchInst>(bb->getTerminator());
287
288 //decrement counter
289 LoadInst* l = new LoadInst(AI, "counter", t);
290
Owen Anderson333c4002009-07-09 23:48:35 +0000291 ICmpInst* s = new ICmpInst(t, ICmpInst::ICMP_EQ, l,
Owen Andersoneed707b2009-07-24 23:12:02 +0000292 ConstantInt::get(T, 0),
Owen Anderson333c4002009-07-09 23:48:35 +0000293 "countercc");
Reid Spencere4d87aa2006-12-23 06:05:41 +0000294
Owen Andersoneed707b2009-07-24 23:12:02 +0000295 Value* nv = BinaryOperator::CreateSub(l, ConstantInt::get(T, 1),
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000296 "counternew", t);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000297 new StoreInst(nv, AI, t);
298 t->setCondition(s);
299
300 //reset counter
301 BasicBlock* oldnext = t->getSuccessor(0);
Owen Anderson1d0be152009-08-13 21:58:54 +0000302 BasicBlock* resetblock = BasicBlock::Create(bb->getContext(),
303 "reset", oldnext->getParent(),
Gabor Greif051a9502008-04-06 20:25:17 +0000304 oldnext);
305 TerminatorInst* t2 = BranchInst::Create(oldnext, resetblock);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000306 t->setSuccessor(0, resetblock);
307 new StoreInst(ResetValue, AI, t2);
308 ReplacePhiPred(oldnext, bb, resetblock);
309}
310
311
312CycleCounter::CycleCounter(Module& m, uint64_t resetmask) : rm(resetmask) {
Duncan Sandse2c43042008-04-07 13:45:04 +0000313 F = Intrinsic::getDeclaration(&m, Intrinsic::readcyclecounter);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000314}
315
316CycleCounter::~CycleCounter() {}
317
318void CycleCounter::PrepFunction(Function* F) {}
319
320void CycleCounter::ProcessChoicePoint(BasicBlock* bb) {
321 BranchInst* t = cast<BranchInst>(bb->getTerminator());
322
Gabor Greif051a9502008-04-06 20:25:17 +0000323 CallInst* c = CallInst::Create(F, "rdcc", t);
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000324 BinaryOperator* b =
Owen Anderson1d0be152009-08-13 21:58:54 +0000325 BinaryOperator::CreateAnd(c,
326 ConstantInt::get(Type::getInt64Ty(bb->getContext()), rm),
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000327 "mrdcc", t);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000328
Owen Anderson333c4002009-07-09 23:48:35 +0000329 ICmpInst *s = new ICmpInst(t, ICmpInst::ICMP_EQ, b,
Owen Anderson1d0be152009-08-13 21:58:54 +0000330 ConstantInt::get(Type::getInt64Ty(bb->getContext()), 0),
Owen Anderson333c4002009-07-09 23:48:35 +0000331 "mrdccc");
Reid Spencere4d87aa2006-12-23 06:05:41 +0000332
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000333 t->setCondition(s);
334}
335
336///////////////////////////////////////
337// Profiling:
338///////////////////////////////////////
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000339bool RSProfilers_std::isProfiling(Value* v) {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000340 if (profcode.find(v) != profcode.end())
341 return true;
342 //else
343 RSProfilers& LI = getAnalysis<RSProfilers>();
344 return LI.isProfiling(v);
345}
346
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000347void RSProfilers_std::IncrementCounterInBlock(BasicBlock *BB, unsigned CounterNum,
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000348 GlobalValue *CounterArray) {
349 // Insert the increment after any alloca or PHI instructions...
Dan Gohman02dea8b2008-05-23 21:05:58 +0000350 BasicBlock::iterator InsertPos = BB->getFirstNonPHI();
351 while (isa<AllocaInst>(InsertPos))
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000352 ++InsertPos;
353
354 // Create the getelementptr constant expression
355 std::vector<Constant*> Indices(2);
Owen Anderson1d0be152009-08-13 21:58:54 +0000356 Indices[0] = Constant::getNullValue(Type::getInt32Ty(BB->getContext()));
357 Indices[1] = ConstantInt::get(Type::getInt32Ty(BB->getContext()), CounterNum);
Owen Andersonbaf3c402009-07-29 18:55:55 +0000358 Constant *ElementPtr =ConstantExpr::getGetElementPtr(CounterArray,
Chris Lattnerec1f7522007-02-19 07:34:47 +0000359 &Indices[0], 2);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000360
361 // Load, increment and store the value back.
362 Value *OldVal = new LoadInst(ElementPtr, "OldCounter", InsertPos);
363 profcode.insert(OldVal);
Gabor Greif7cbd8a32008-05-16 19:29:10 +0000364 Value *NewVal = BinaryOperator::CreateAdd(OldVal,
Owen Anderson1d0be152009-08-13 21:58:54 +0000365 ConstantInt::get(Type::getInt32Ty(BB->getContext()), 1),
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000366 "NewCounter", InsertPos);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000367 profcode.insert(NewVal);
368 profcode.insert(new StoreInst(NewVal, ElementPtr, InsertPos));
369}
370
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000371void RSProfilers_std::getAnalysisUsage(AnalysisUsage &AU) const {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000372 //grab any outstanding profiler, or get the null one
373 AU.addRequired<RSProfilers>();
374}
375
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000376///////////////////////////////////////
377// RS Framework
378///////////////////////////////////////
379
380Value* ProfilerRS::Translate(Value* v) {
381 if(TransCache[v])
382 return TransCache[v];
383
384 if (BasicBlock* bb = dyn_cast<BasicBlock>(v)) {
385 if (bb == &bb->getParent()->getEntryBlock())
386 TransCache[bb] = bb; //don't translate entry block
387 else
Owen Anderson1d0be152009-08-13 21:58:54 +0000388 TransCache[bb] = BasicBlock::Create(v->getContext(),
389 "dup_" + bb->getName(),
Gabor Greifb1dbcd82008-05-15 10:04:30 +0000390 bb->getParent(), NULL);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000391 return TransCache[bb];
392 } else if (Instruction* i = dyn_cast<Instruction>(v)) {
393 //we have already translated this
394 //do not translate entry block allocas
395 if(&i->getParent()->getParent()->getEntryBlock() == i->getParent()) {
396 TransCache[i] = i;
397 return i;
398 } else {
399 //translate this
Nick Lewycky67760642009-09-27 07:38:41 +0000400 Instruction* i2 = i->clone();
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000401 if (i->hasName())
402 i2->setName("dup_" + i->getName());
403 TransCache[i] = i2;
404 //NumNewInst++;
405 for (unsigned x = 0; x < i2->getNumOperands(); ++x)
406 i2->setOperand(x, Translate(i2->getOperand(x)));
407 return i2;
408 }
409 } else if (isa<Function>(v) || isa<Constant>(v) || isa<Argument>(v)) {
410 TransCache[v] = v;
411 return v;
412 }
Torok Edwinc23197a2009-07-14 16:55:14 +0000413 llvm_unreachable("Value not handled");
Jeff Cohen3523f6e2005-11-28 06:45:57 +0000414 return 0;
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000415}
416
417void ProfilerRS::Duplicate(Function& F, RSProfilers& LI)
418{
419 //perform a breadth first search, building up a duplicate of the code
420 std::queue<BasicBlock*> worklist;
421 std::set<BasicBlock*> seen;
422
423 //This loop ensures proper BB order, to help performance
424 for (Function::iterator fib = F.begin(), fie = F.end(); fib != fie; ++fib)
425 worklist.push(fib);
426 while (!worklist.empty()) {
427 Translate(worklist.front());
428 worklist.pop();
429 }
430
431 //remember than reg2mem created a new entry block we don't want to duplicate
432 worklist.push(F.getEntryBlock().getTerminator()->getSuccessor(0));
433 seen.insert(&F.getEntryBlock());
434
435 while (!worklist.empty()) {
436 BasicBlock* bb = worklist.front();
437 worklist.pop();
438 if(seen.find(bb) == seen.end()) {
439 BasicBlock* bbtarget = cast<BasicBlock>(Translate(bb));
440 BasicBlock::InstListType& instlist = bbtarget->getInstList();
441 for (BasicBlock::iterator iib = bb->begin(), iie = bb->end();
442 iib != iie; ++iib) {
443 //NumOldInst++;
444 if (!LI.isProfiling(&*iib)) {
445 Instruction* i = cast<Instruction>(Translate(iib));
446 instlist.insert(bbtarget->end(), i);
447 }
448 }
449 //updated search state;
450 seen.insert(bb);
451 TerminatorInst* ti = bb->getTerminator();
452 for (unsigned x = 0; x < ti->getNumSuccessors(); ++x) {
453 BasicBlock* bbs = ti->getSuccessor(x);
454 if (seen.find(bbs) == seen.end()) {
455 worklist.push(bbs);
456 }
457 }
458 }
459 }
460}
461
462void ProfilerRS::ProcessBackEdge(BasicBlock* src, BasicBlock* dst, Function& F) {
463 //given a backedge from B -> A, and translations A' and B',
464 //a: insert C and C'
465 //b: add branches in C to A and A' and in C' to A and A'
466 //c: mod terminators@B, replace A with C
467 //d: mod terminators@B', replace A' with C'
468 //e: mod phis@A for pred B to be pred C
469 // if multiple entries, simplify to one
470 //f: mod phis@A' for pred B' to be pred C'
471 // if multiple entries, simplify to one
472 //g: for all phis@A with pred C using x
473 // add in edge from C' using x'
474 // add in edge from C using x in A'
475
476 //a:
Chris Lattnere24c92a2007-04-17 17:54:12 +0000477 Function::iterator BBN = src; ++BBN;
Owen Anderson1d0be152009-08-13 21:58:54 +0000478 BasicBlock* bbC = BasicBlock::Create(F.getContext(), "choice", &F, BBN);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000479 //ChoicePoints.insert(bbC);
Chris Lattnere24c92a2007-04-17 17:54:12 +0000480 BBN = cast<BasicBlock>(Translate(src));
Owen Anderson1d0be152009-08-13 21:58:54 +0000481 BasicBlock* bbCp = BasicBlock::Create(F.getContext(), "choice", &F, ++BBN);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000482 ChoicePoints.insert(bbCp);
483
484 //b:
Gabor Greif051a9502008-04-06 20:25:17 +0000485 BranchInst::Create(cast<BasicBlock>(Translate(dst)), bbC);
486 BranchInst::Create(dst, cast<BasicBlock>(Translate(dst)),
Owen Anderson1d0be152009-08-13 21:58:54 +0000487 ConstantInt::get(Type::getInt1Ty(src->getContext()), true), bbCp);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000488 //c:
489 {
490 TerminatorInst* iB = src->getTerminator();
491 for (unsigned x = 0; x < iB->getNumSuccessors(); ++x)
492 if (iB->getSuccessor(x) == dst)
493 iB->setSuccessor(x, bbC);
494 }
495 //d:
496 {
497 TerminatorInst* iBp = cast<TerminatorInst>(Translate(src->getTerminator()));
498 for (unsigned x = 0; x < iBp->getNumSuccessors(); ++x)
499 if (iBp->getSuccessor(x) == cast<BasicBlock>(Translate(dst)))
500 iBp->setSuccessor(x, bbCp);
501 }
502 //e:
503 ReplacePhiPred(dst, src, bbC);
504 //src could be a switch, in which case we are replacing several edges with one
505 //thus collapse those edges int the Phi
506 CollapsePhi(dst, bbC);
507 //f:
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000508 ReplacePhiPred(cast<BasicBlock>(Translate(dst)),
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000509 cast<BasicBlock>(Translate(src)),bbCp);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000510 CollapsePhi(cast<BasicBlock>(Translate(dst)), bbCp);
511 //g:
512 for(BasicBlock::iterator ib = dst->begin(), ie = dst->end(); ib != ie;
513 ++ib)
514 if (PHINode* phi = dyn_cast<PHINode>(&*ib)) {
515 for(unsigned x = 0; x < phi->getNumIncomingValues(); ++x)
516 if(bbC == phi->getIncomingBlock(x)) {
517 phi->addIncoming(Translate(phi->getIncomingValue(x)), bbCp);
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000518 cast<PHINode>(Translate(phi))->addIncoming(phi->getIncomingValue(x),
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000519 bbC);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000520 }
521 phi->removeIncomingValue(bbC);
522 }
523}
524
525bool ProfilerRS::runOnFunction(Function& F) {
Reid Spencer5cbf9852007-01-30 20:08:39 +0000526 if (!F.isDeclaration()) {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000527 std::set<std::pair<BasicBlock*, BasicBlock*> > BackEdges;
528 RSProfilers& LI = getAnalysis<RSProfilers>();
529
530 getBackEdges(F, BackEdges);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000531 Duplicate(F, LI);
532 //assume that stuff worked. now connect the duplicated basic blocks
533 //with the originals in such a way as to preserve ssa. yuk!
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000534 for (std::set<std::pair<BasicBlock*, BasicBlock*> >::iterator
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000535 ib = BackEdges.begin(), ie = BackEdges.end(); ib != ie; ++ib)
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000536 ProcessBackEdge(ib->first, ib->second, F);
537
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000538 //oh, and add the edge from the reg2mem created entry node to the
539 //duplicated second node
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000540 TerminatorInst* T = F.getEntryBlock().getTerminator();
Gabor Greif051a9502008-04-06 20:25:17 +0000541 ReplaceInstWithInst(T, BranchInst::Create(T->getSuccessor(0),
542 cast<BasicBlock>(
Owen Anderson1d0be152009-08-13 21:58:54 +0000543 Translate(T->getSuccessor(0))),
544 ConstantInt::get(Type::getInt1Ty(F.getContext()), true)));
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000545
546 //do whatever is needed now that the function is duplicated
547 c->PrepFunction(&F);
548
549 //add entry node to choice points
550 ChoicePoints.insert(&F.getEntryBlock());
551
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000552 for (std::set<BasicBlock*>::iterator
Anton Korobeynikovbed29462007-04-16 18:10:23 +0000553 ii = ChoicePoints.begin(), ie = ChoicePoints.end(); ii != ie; ++ii)
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000554 c->ProcessChoicePoint(*ii);
555
556 ChoicePoints.clear();
557 TransCache.clear();
558
559 return true;
560 }
561 return false;
562}
563
564bool ProfilerRS::doInitialization(Module &M) {
565 switch (RandomMethod) {
566 case GBV:
Owen Anderson1d0be152009-08-13 21:58:54 +0000567 c = new GlobalRandomCounter(M, Type::getInt32Ty(M.getContext()),
568 (1 << 14) - 1);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000569 break;
570 case GBVO:
Owen Anderson1d0be152009-08-13 21:58:54 +0000571 c = new GlobalRandomCounterOpt(M, Type::getInt32Ty(M.getContext()),
572 (1 << 14) - 1);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000573 break;
574 case HOSTCC:
575 c = new CycleCounter(M, (1 << 14) - 1);
576 break;
577 };
578 return true;
579}
580
581void ProfilerRS::getAnalysisUsage(AnalysisUsage &AU) const {
582 AU.addRequired<RSProfilers>();
583 AU.addRequiredID(DemoteRegisterToMemoryID);
584}
585
586///////////////////////////////////////
587// Utilities:
588///////////////////////////////////////
589static void ReplacePhiPred(BasicBlock* btarget,
590 BasicBlock* bold, BasicBlock* bnew) {
591 for(BasicBlock::iterator ib = btarget->begin(), ie = btarget->end();
592 ib != ie; ++ib)
593 if (PHINode* phi = dyn_cast<PHINode>(&*ib)) {
594 for(unsigned x = 0; x < phi->getNumIncomingValues(); ++x)
595 if(bold == phi->getIncomingBlock(x))
596 phi->setIncomingBlock(x, bnew);
597 }
598}
599
600static void CollapsePhi(BasicBlock* btarget, BasicBlock* bsrc) {
601 for(BasicBlock::iterator ib = btarget->begin(), ie = btarget->end();
602 ib != ie; ++ib)
603 if (PHINode* phi = dyn_cast<PHINode>(&*ib)) {
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000604 std::map<BasicBlock*, Value*> counter;
605 for(unsigned i = 0; i < phi->getNumIncomingValues(); ) {
606 if (counter[phi->getIncomingBlock(i)]) {
Andrew Lenharthbb227c12005-11-28 18:00:38 +0000607 assert(phi->getIncomingValue(i) == counter[phi->getIncomingBlock(i)]);
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000608 phi->removeIncomingValue(i, false);
609 } else {
610 counter[phi->getIncomingBlock(i)] = phi->getIncomingValue(i);
611 ++i;
612 }
613 }
614 }
615}
616
617template<class T>
618static void recBackEdge(BasicBlock* bb, T& BackEdges,
619 std::map<BasicBlock*, int>& color,
620 std::map<BasicBlock*, int>& depth,
621 std::map<BasicBlock*, int>& finish,
622 int& time)
623{
624 color[bb] = 1;
625 ++time;
626 depth[bb] = time;
627 TerminatorInst* t= bb->getTerminator();
628 for(unsigned i = 0; i < t->getNumSuccessors(); ++i) {
629 BasicBlock* bbnew = t->getSuccessor(i);
630 if (color[bbnew] == 0)
631 recBackEdge(bbnew, BackEdges, color, depth, finish, time);
632 else if (color[bbnew] == 1) {
633 BackEdges.insert(std::make_pair(bb, bbnew));
634 //NumBackEdges++;
635 }
636 }
637 color[bb] = 2;
638 ++time;
639 finish[bb] = time;
640}
641
642
643
644//find the back edges and where they go to
645template<class T>
646static void getBackEdges(Function& F, T& BackEdges) {
647 std::map<BasicBlock*, int> color;
648 std::map<BasicBlock*, int> depth;
649 std::map<BasicBlock*, int> finish;
650 int time = 0;
651 recBackEdge(&F.getEntryBlock(), BackEdges, color, depth, finish, time);
Daniel Dunbarce63ffb2009-07-25 00:23:56 +0000652 DEBUG(errs() << F.getName() << " " << BackEdges.size() << "\n");
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000653}
654
655
656//Creation functions
Andrew Lenharth701f5ac2005-11-28 00:58:09 +0000657ModulePass* llvm::createNullProfilerRSPass() {
658 return new NullProfilerRS();
659}
660
661FunctionPass* llvm::createRSProfilingPass() {
662 return new ProfilerRS();
663}