blob: 6f994eb2c7a6fa43d8f783291af61302d04e765c [file] [log] [blame]
Rong Xuf430ae42015-12-09 18:08:16 +00001//===-- PGOInstrumentation.cpp - MST-based PGO Instrumentation ------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements PGO instrumentation using a minimum spanning tree based
11// on the following paper:
12// [1] Donald E. Knuth, Francis R. Stevenson. Optimal measurement of points
13// for program frequency counts. BIT Numerical Mathematics 1973, Volume 13,
14// Issue 3, pp 313-322
15// The idea of the algorithm based on the fact that for each node (except for
16// the entry and exit), the sum of incoming edge counts equals the sum of
17// outgoing edge counts. The count of edge on spanning tree can be derived from
18// those edges not on the spanning tree. Knuth proves this method instruments
19// the minimum number of edges.
20//
21// The minimal spanning tree here is actually a maximum weight tree -- on-tree
22// edges have higher frequencies (more likely to execute). The idea is to
23// instrument those less frequently executed edges to reduce the runtime
24// overhead of instrumented binaries.
25//
26// This file contains two passes:
27// (1) Pass PGOInstrumentationGen which instruments the IR to generate edge
Rong Xu13b01dc2016-02-10 18:24:45 +000028// count profile, and generates the instrumentation for indirect call
29// profiling.
Rong Xuf430ae42015-12-09 18:08:16 +000030// (2) Pass PGOInstrumentationUse which reads the edge count profile and
Rong Xu13b01dc2016-02-10 18:24:45 +000031// annotates the branch weights. It also reads the indirect call value
32// profiling records and annotate the indirect call instructions.
33//
Rong Xuf430ae42015-12-09 18:08:16 +000034// To get the precise counter information, These two passes need to invoke at
35// the same compilation point (so they see the same IR). For pass
36// PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For
37// pass PGOInstrumentationUse, the real work in done in class PGOUseFunc and
38// the profile is opened in module level and passed to each PGOUseFunc instance.
39// The shared code for PGOInstrumentationGen and PGOInstrumentationUse is put
40// in class FuncPGOInstrumentation.
41//
42// Class PGOEdge represents a CFG edge and some auxiliary information. Class
43// BBInfo contains auxiliary information for each BB. These two classes are used
44// in pass PGOInstrumentationGen. Class PGOUseEdge and UseBBInfo are the derived
45// class of PGOEdge and BBInfo, respectively. They contains extra data structure
46// used in populating profile counters.
47// The MST implementation is in Class CFGMST (CFGMST.h).
48//
49//===----------------------------------------------------------------------===//
50
Xinliang David Li8aebf442016-05-06 05:49:19 +000051#include "llvm/Transforms/PGOInstrumentation.h"
Rong Xuf430ae42015-12-09 18:08:16 +000052#include "CFGMST.h"
Rong Xu0eb36032016-04-01 23:16:44 +000053#include "IndirectCallSiteVisitor.h"
Rong Xuf430ae42015-12-09 18:08:16 +000054#include "llvm/ADT/STLExtras.h"
55#include "llvm/ADT/Statistic.h"
Rong Xu33c76c02016-02-10 17:18:30 +000056#include "llvm/ADT/Triple.h"
Rong Xuf430ae42015-12-09 18:08:16 +000057#include "llvm/Analysis/BlockFrequencyInfo.h"
58#include "llvm/Analysis/BranchProbabilityInfo.h"
59#include "llvm/Analysis/CFG.h"
Rong Xued9fec72016-01-21 18:11:44 +000060#include "llvm/IR/CallSite.h"
Rong Xuf430ae42015-12-09 18:08:16 +000061#include "llvm/IR/DiagnosticInfo.h"
62#include "llvm/IR/IRBuilder.h"
63#include "llvm/IR/InstIterator.h"
64#include "llvm/IR/Instructions.h"
65#include "llvm/IR/IntrinsicInst.h"
66#include "llvm/IR/MDBuilder.h"
67#include "llvm/IR/Module.h"
68#include "llvm/Pass.h"
69#include "llvm/ProfileData/InstrProfReader.h"
Easwaran Raman5fe04a12016-05-26 22:57:11 +000070#include "llvm/ProfileData/ProfileCommon.h"
Rong Xuf430ae42015-12-09 18:08:16 +000071#include "llvm/Support/BranchProbability.h"
72#include "llvm/Support/Debug.h"
73#include "llvm/Support/JamCRC.h"
Rong Xued9fec72016-01-21 18:11:44 +000074#include "llvm/Transforms/Instrumentation.h"
Rong Xuf430ae42015-12-09 18:08:16 +000075#include "llvm/Transforms/Utils/BasicBlockUtils.h"
Xinliang David Li8aebf442016-05-06 05:49:19 +000076#include <algorithm>
Rong Xuf430ae42015-12-09 18:08:16 +000077#include <string>
78#include <utility>
79#include <vector>
80
81using namespace llvm;
82
83#define DEBUG_TYPE "pgo-instrumentation"
84
85STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.");
86STATISTIC(NumOfPGOEdge, "Number of edges.");
87STATISTIC(NumOfPGOBB, "Number of basic-blocks.");
88STATISTIC(NumOfPGOSplit, "Number of critical edge splits.");
89STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.");
90STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");
91STATISTIC(NumOfPGOMissing, "Number of functions without profile.");
Rong Xu13b01dc2016-02-10 18:24:45 +000092STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.");
Rong Xuf430ae42015-12-09 18:08:16 +000093
94// Command line option to specify the file to read profile from. This is
95// mainly used for testing.
96static cl::opt<std::string>
97 PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden,
98 cl::value_desc("filename"),
99 cl::desc("Specify the path of profile data file. This is"
100 "mainly for test purpose."));
101
Rong Xuecdc98f2016-03-04 22:08:44 +0000102// Command line option to disable value profiling. The default is false:
Rong Xu13b01dc2016-02-10 18:24:45 +0000103// i.e. value profiling is enabled by default. This is for debug purpose.
Rong Xu9e926e82016-02-29 19:16:04 +0000104static cl::opt<bool> DisableValueProfiling("disable-vp", cl::init(false),
105 cl::Hidden,
106 cl::desc("Disable Value Profiling"));
Rong Xued9fec72016-01-21 18:11:44 +0000107
Rong Xuecdc98f2016-03-04 22:08:44 +0000108// Command line option to set the maximum number of VP annotations to write to
Rong Xu08afb052016-04-28 17:31:22 +0000109// the metadata for a single indirect call callsite.
110static cl::opt<unsigned> MaxNumAnnotations(
111 "icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore,
112 cl::desc("Max number of annotations for a single indirect "
113 "call callsite"));
Rong Xuecdc98f2016-03-04 22:08:44 +0000114
Rong Xu0698de92016-05-13 17:26:06 +0000115// Command line option to enable/disable the warning about missing profile
116// information.
117static cl::opt<bool> NoPGOWarnMissing("no-pgo-warn-missing", cl::init(false),
118 cl::Hidden);
119
120// Command line option to enable/disable the warning about a hash mismatch in
121// the profile data.
122static cl::opt<bool> NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false),
123 cl::Hidden);
124
Rong Xuf430ae42015-12-09 18:08:16 +0000125namespace {
Xinliang David Li8aebf442016-05-06 05:49:19 +0000126class PGOInstrumentationGenLegacyPass : public ModulePass {
Rong Xuf430ae42015-12-09 18:08:16 +0000127public:
128 static char ID;
129
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000130 PGOInstrumentationGenLegacyPass() : ModulePass(ID) {
Xinliang David Li8aebf442016-05-06 05:49:19 +0000131 initializePGOInstrumentationGenLegacyPassPass(
132 *PassRegistry::getPassRegistry());
Rong Xuf430ae42015-12-09 18:08:16 +0000133 }
134
135 const char *getPassName() const override {
136 return "PGOInstrumentationGenPass";
137 }
138
139private:
140 bool runOnModule(Module &M) override;
141
142 void getAnalysisUsage(AnalysisUsage &AU) const override {
143 AU.addRequired<BlockFrequencyInfoWrapperPass>();
144 }
145};
146
Xinliang David Lid55827f2016-05-07 05:39:12 +0000147class PGOInstrumentationUseLegacyPass : public ModulePass {
Rong Xuf430ae42015-12-09 18:08:16 +0000148public:
149 static char ID;
150
151 // Provide the profile filename as the parameter.
Xinliang David Lid55827f2016-05-07 05:39:12 +0000152 PGOInstrumentationUseLegacyPass(std::string Filename = "")
Benjamin Kramer82de7d32016-05-27 14:27:24 +0000153 : ModulePass(ID), ProfileFileName(std::move(Filename)) {
Rong Xuf430ae42015-12-09 18:08:16 +0000154 if (!PGOTestProfileFile.empty())
155 ProfileFileName = PGOTestProfileFile;
Xinliang David Lid55827f2016-05-07 05:39:12 +0000156 initializePGOInstrumentationUseLegacyPassPass(
157 *PassRegistry::getPassRegistry());
Rong Xuf430ae42015-12-09 18:08:16 +0000158 }
159
160 const char *getPassName() const override {
161 return "PGOInstrumentationUsePass";
162 }
163
164private:
165 std::string ProfileFileName;
Rong Xuf430ae42015-12-09 18:08:16 +0000166
Xinliang David Lida195582016-05-10 21:59:52 +0000167 bool runOnModule(Module &M) override;
Rong Xuf430ae42015-12-09 18:08:16 +0000168 void getAnalysisUsage(AnalysisUsage &AU) const override {
169 AU.addRequired<BlockFrequencyInfoWrapperPass>();
170 }
171};
172} // end anonymous namespace
173
Xinliang David Li8aebf442016-05-06 05:49:19 +0000174char PGOInstrumentationGenLegacyPass::ID = 0;
175INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",
Rong Xuf430ae42015-12-09 18:08:16 +0000176 "PGO instrumentation.", false, false)
177INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
178INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
Xinliang David Li8aebf442016-05-06 05:49:19 +0000179INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",
Rong Xuf430ae42015-12-09 18:08:16 +0000180 "PGO instrumentation.", false, false)
181
Xinliang David Li8aebf442016-05-06 05:49:19 +0000182ModulePass *llvm::createPGOInstrumentationGenLegacyPass() {
183 return new PGOInstrumentationGenLegacyPass();
Rong Xuf430ae42015-12-09 18:08:16 +0000184}
185
Xinliang David Lid55827f2016-05-07 05:39:12 +0000186char PGOInstrumentationUseLegacyPass::ID = 0;
187INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use",
Rong Xuf430ae42015-12-09 18:08:16 +0000188 "Read PGO instrumentation profile.", false, false)
189INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
190INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
Xinliang David Lid55827f2016-05-07 05:39:12 +0000191INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use",
Rong Xuf430ae42015-12-09 18:08:16 +0000192 "Read PGO instrumentation profile.", false, false)
193
Xinliang David Lid55827f2016-05-07 05:39:12 +0000194ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) {
195 return new PGOInstrumentationUseLegacyPass(Filename.str());
Rong Xuf430ae42015-12-09 18:08:16 +0000196}
197
198namespace {
199/// \brief An MST based instrumentation for PGO
200///
201/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
202/// in the function level.
203struct PGOEdge {
204 // This class implements the CFG edges. Note the CFG can be a multi-graph.
205 // So there might be multiple edges with same SrcBB and DestBB.
206 const BasicBlock *SrcBB;
207 const BasicBlock *DestBB;
208 uint64_t Weight;
209 bool InMST;
210 bool Removed;
211 bool IsCritical;
212 PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
213 : SrcBB(Src), DestBB(Dest), Weight(W), InMST(false), Removed(false),
214 IsCritical(false) {}
215 // Return the information string of an edge.
216 const std::string infoString() const {
217 return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
218 (IsCritical ? "c" : " ") + " W=" + Twine(Weight)).str();
219 }
220};
221
222// This class stores the auxiliary information for each BB.
223struct BBInfo {
224 BBInfo *Group;
225 uint32_t Index;
226 uint32_t Rank;
227
228 BBInfo(unsigned IX) : Group(this), Index(IX), Rank(0) {}
229
230 // Return the information string of this object.
231 const std::string infoString() const {
232 return (Twine("Index=") + Twine(Index)).str();
233 }
234};
235
236// This class implements the CFG edges. Note the CFG can be a multi-graph.
237template <class Edge, class BBInfo> class FuncPGOInstrumentation {
238private:
239 Function &F;
240 void computeCFGHash();
241
242public:
243 std::string FuncName;
244 GlobalVariable *FuncNameVar;
245 // CFG hash value for this function.
246 uint64_t FunctionHash;
247
248 // The Minimum Spanning Tree of function CFG.
249 CFGMST<Edge, BBInfo> MST;
250
251 // Give an edge, find the BB that will be instrumented.
252 // Return nullptr if there is no BB to be instrumented.
253 BasicBlock *getInstrBB(Edge *E);
254
255 // Return the auxiliary BB information.
256 BBInfo &getBBInfo(const BasicBlock *BB) const { return MST.getBBInfo(BB); }
257
258 // Dump edges and BB information.
259 void dumpInfo(std::string Str = "") const {
260 MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName + " Hash: " +
Rong Xued9fec72016-01-21 18:11:44 +0000261 Twine(FunctionHash) + "\t" + Str);
Rong Xuf430ae42015-12-09 18:08:16 +0000262 }
263
264 FuncPGOInstrumentation(Function &Func, bool CreateGlobalVar = false,
265 BranchProbabilityInfo *BPI = nullptr,
266 BlockFrequencyInfo *BFI = nullptr)
267 : F(Func), FunctionHash(0), MST(F, BPI, BFI) {
268 FuncName = getPGOFuncName(F);
269 computeCFGHash();
270 DEBUG(dumpInfo("after CFGMST"));
271
272 NumOfPGOBB += MST.BBInfos.size();
273 for (auto &E : MST.AllEdges) {
274 if (E->Removed)
275 continue;
276 NumOfPGOEdge++;
277 if (!E->InMST)
278 NumOfPGOInstrument++;
279 }
280
281 if (CreateGlobalVar)
282 FuncNameVar = createPGOFuncNameVar(F, FuncName);
Eugene Zelenko6ac3f732016-01-26 18:48:36 +0000283 }
Rong Xuf430ae42015-12-09 18:08:16 +0000284};
285
286// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
287// value of each BB in the CFG. The higher 32 bits record the number of edges.
288template <class Edge, class BBInfo>
289void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
290 std::vector<char> Indexes;
291 JamCRC JC;
292 for (auto &BB : F) {
293 const TerminatorInst *TI = BB.getTerminator();
294 for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
295 BasicBlock *Succ = TI->getSuccessor(I);
296 uint32_t Index = getBBInfo(Succ).Index;
297 for (int J = 0; J < 4; J++)
298 Indexes.push_back((char)(Index >> (J * 8)));
299 }
300 }
301 JC.update(Indexes);
302 FunctionHash = (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
303}
304
305// Given a CFG E to be instrumented, find which BB to place the instrumented
306// code. The function will split the critical edge if necessary.
307template <class Edge, class BBInfo>
308BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
309 if (E->InMST || E->Removed)
310 return nullptr;
311
312 BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB);
313 BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB);
314 // For a fake edge, instrument the real BB.
315 if (SrcBB == nullptr)
316 return DestBB;
317 if (DestBB == nullptr)
318 return SrcBB;
319
320 // Instrument the SrcBB if it has a single successor,
321 // otherwise, the DestBB if this is not a critical edge.
322 TerminatorInst *TI = SrcBB->getTerminator();
323 if (TI->getNumSuccessors() <= 1)
324 return SrcBB;
325 if (!E->IsCritical)
326 return DestBB;
327
328 // For a critical edge, we have to split. Instrument the newly
329 // created BB.
330 NumOfPGOSplit++;
331 DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index << " --> "
332 << getBBInfo(DestBB).Index << "\n");
333 unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
334 BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum);
335 assert(InstrBB && "Critical edge is not split");
336
337 E->Removed = true;
338 return InstrBB;
339}
340
Rong Xued9fec72016-01-21 18:11:44 +0000341// Visit all edge and instrument the edges not in MST, and do value profiling.
Rong Xuf430ae42015-12-09 18:08:16 +0000342// Critical edges will be split.
343static void instrumentOneFunc(Function &F, Module *M,
344 BranchProbabilityInfo *BPI,
345 BlockFrequencyInfo *BFI) {
346 unsigned NumCounters = 0;
347 FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, true, BPI, BFI);
348 for (auto &E : FuncInfo.MST.AllEdges) {
349 if (!E->InMST && !E->Removed)
350 NumCounters++;
351 }
352
353 uint32_t I = 0;
Rong Xued9fec72016-01-21 18:11:44 +0000354 Type *I8PtrTy = Type::getInt8PtrTy(M->getContext());
Rong Xuf430ae42015-12-09 18:08:16 +0000355 for (auto &E : FuncInfo.MST.AllEdges) {
356 BasicBlock *InstrBB = FuncInfo.getInstrBB(E.get());
357 if (!InstrBB)
358 continue;
359
360 IRBuilder<> Builder(InstrBB, InstrBB->getFirstInsertionPt());
361 assert(Builder.GetInsertPoint() != InstrBB->end() &&
362 "Cannot get the Instrumentation point");
Rong Xuf430ae42015-12-09 18:08:16 +0000363 Builder.CreateCall(
364 Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment),
365 {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
366 Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters),
367 Builder.getInt32(I++)});
368 }
Rong Xued9fec72016-01-21 18:11:44 +0000369
370 if (DisableValueProfiling)
371 return;
372
373 unsigned NumIndirectCallSites = 0;
Rong Xu0eb36032016-04-01 23:16:44 +0000374 for (auto &I : findIndirectCallSites(F)) {
Rong Xued9fec72016-01-21 18:11:44 +0000375 CallSite CS(I);
376 Value *Callee = CS.getCalledValue();
377 DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = "
378 << NumIndirectCallSites << "\n");
379 IRBuilder<> Builder(I);
380 assert(Builder.GetInsertPoint() != I->getParent()->end() &&
381 "Cannot get the Instrumentation point");
382 Builder.CreateCall(
383 Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
384 {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
385 Builder.getInt64(FuncInfo.FunctionHash),
386 Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()),
387 Builder.getInt32(llvm::InstrProfValueKind::IPVK_IndirectCallTarget),
388 Builder.getInt32(NumIndirectCallSites++)});
389 }
390 NumOfPGOICall += NumIndirectCallSites;
Rong Xuf430ae42015-12-09 18:08:16 +0000391}
392
393// This class represents a CFG edge in profile use compilation.
394struct PGOUseEdge : public PGOEdge {
395 bool CountValid;
396 uint64_t CountValue;
397 PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
398 : PGOEdge(Src, Dest, W), CountValid(false), CountValue(0) {}
399
400 // Set edge count value
401 void setEdgeCount(uint64_t Value) {
402 CountValue = Value;
403 CountValid = true;
404 }
405
406 // Return the information string for this object.
407 const std::string infoString() const {
408 if (!CountValid)
409 return PGOEdge::infoString();
Rong Xu9e926e82016-02-29 19:16:04 +0000410 return (Twine(PGOEdge::infoString()) + " Count=" + Twine(CountValue))
411 .str();
Rong Xuf430ae42015-12-09 18:08:16 +0000412 }
413};
414
415typedef SmallVector<PGOUseEdge *, 2> DirectEdges;
416
417// This class stores the auxiliary information for each BB.
418struct UseBBInfo : public BBInfo {
419 uint64_t CountValue;
420 bool CountValid;
421 int32_t UnknownCountInEdge;
422 int32_t UnknownCountOutEdge;
423 DirectEdges InEdges;
424 DirectEdges OutEdges;
425 UseBBInfo(unsigned IX)
426 : BBInfo(IX), CountValue(0), CountValid(false), UnknownCountInEdge(0),
427 UnknownCountOutEdge(0) {}
428 UseBBInfo(unsigned IX, uint64_t C)
429 : BBInfo(IX), CountValue(C), CountValid(true), UnknownCountInEdge(0),
430 UnknownCountOutEdge(0) {}
431
432 // Set the profile count value for this BB.
433 void setBBInfoCount(uint64_t Value) {
434 CountValue = Value;
435 CountValid = true;
436 }
437
438 // Return the information string of this object.
439 const std::string infoString() const {
440 if (!CountValid)
441 return BBInfo::infoString();
442 return (Twine(BBInfo::infoString()) + " Count=" + Twine(CountValue)).str();
443 }
444};
445
446// Sum up the count values for all the edges.
447static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
448 uint64_t Total = 0;
449 for (auto &E : Edges) {
450 if (E->Removed)
451 continue;
452 Total += E->CountValue;
453 }
454 return Total;
455}
456
457class PGOUseFunc {
Rong Xu6090afd2016-03-28 17:08:56 +0000458public:
459 PGOUseFunc(Function &Func, Module *Modu, BranchProbabilityInfo *BPI = nullptr,
460 BlockFrequencyInfo *BFI = nullptr)
Sean Silva2a730192016-05-28 03:02:50 +0000461 : F(Func), M(Modu), FuncInfo(Func, false, BPI, BFI) {}
Rong Xu6090afd2016-03-28 17:08:56 +0000462
463 // Read counts for the instrumented BB from profile.
464 bool readCounters(IndexedInstrProfReader *PGOReader);
465
466 // Populate the counts for all BBs.
467 void populateCounters();
468
469 // Set the branch weights based on the count values.
470 void setBranchWeights();
471
472 // Annotate the indirect call sites.
473 void annotateIndirectCallSites();
474
Easwaran Raman5fe04a12016-05-26 22:57:11 +0000475 // Return the profile record for this function;
476 InstrProfRecord &getProfileRecord() { return ProfileRecord; }
477
Sean Silva2a730192016-05-28 03:02:50 +0000478 // The entry count of this function.
479 uint64_t EntryCount;
480
481 // The maximum count value of any BB in this function.
482 uint64_t MaxBBCount;
483
Rong Xuf430ae42015-12-09 18:08:16 +0000484private:
485 Function &F;
486 Module *M;
487 // This member stores the shared information with class PGOGenFunc.
488 FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
489
490 // Return the auxiliary BB information.
491 UseBBInfo &getBBInfo(const BasicBlock *BB) const {
492 return FuncInfo.getBBInfo(BB);
493 }
494
Rong Xu13b01dc2016-02-10 18:24:45 +0000495 // ProfileRecord for this function.
496 InstrProfRecord ProfileRecord;
497
Rong Xuf430ae42015-12-09 18:08:16 +0000498 // Find the Instrumented BB and set the value.
499 void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
500
501 // Set the edge counter value for the unknown edge -- there should be only
502 // one unknown edge.
503 void setEdgeCount(DirectEdges &Edges, uint64_t Value);
504
505 // Return FuncName string;
506 const std::string getFuncName() const { return FuncInfo.FuncName; }
Rong Xuf430ae42015-12-09 18:08:16 +0000507};
508
509// Visit all the edges and assign the count value for the instrumented
510// edges and the BB.
511void PGOUseFunc::setInstrumentedCounts(
512 const std::vector<uint64_t> &CountFromProfile) {
513
514 // Use a worklist as we will update the vector during the iteration.
515 std::vector<PGOUseEdge *> WorkList;
516 for (auto &E : FuncInfo.MST.AllEdges)
517 WorkList.push_back(E.get());
518
519 uint32_t I = 0;
520 for (auto &E : WorkList) {
521 BasicBlock *InstrBB = FuncInfo.getInstrBB(E);
522 if (!InstrBB)
523 continue;
524 uint64_t CountValue = CountFromProfile[I++];
525 if (!E->Removed) {
526 getBBInfo(InstrBB).setBBInfoCount(CountValue);
527 E->setEdgeCount(CountValue);
528 continue;
529 }
530
531 // Need to add two new edges.
532 BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB);
533 BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB);
534 // Add new edge of SrcBB->InstrBB.
535 PGOUseEdge &NewEdge = FuncInfo.MST.addEdge(SrcBB, InstrBB, 0);
536 NewEdge.setEdgeCount(CountValue);
537 // Add new edge of InstrBB->DestBB.
538 PGOUseEdge &NewEdge1 = FuncInfo.MST.addEdge(InstrBB, DestBB, 0);
539 NewEdge1.setEdgeCount(CountValue);
540 NewEdge1.InMST = true;
541 getBBInfo(InstrBB).setBBInfoCount(CountValue);
542 }
543}
544
545// Set the count value for the unknown edge. There should be one and only one
546// unknown edge in Edges vector.
547void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
548 for (auto &E : Edges) {
549 if (E->CountValid)
550 continue;
551 E->setEdgeCount(Value);
552
553 getBBInfo(E->SrcBB).UnknownCountOutEdge--;
554 getBBInfo(E->DestBB).UnknownCountInEdge--;
555 return;
556 }
557 llvm_unreachable("Cannot find the unknown count edge");
558}
559
560// Read the profile from ProfileFileName and assign the value to the
561// instrumented BB and the edges. This function also updates ProgramMaxCount.
562// Return true if the profile are successfully read, and false on errors.
563bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) {
564 auto &Ctx = M->getContext();
Vedant Kumar9152fd12016-05-19 03:54:45 +0000565 Expected<InstrProfRecord> Result =
Rong Xuf430ae42015-12-09 18:08:16 +0000566 PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash);
Vedant Kumar9152fd12016-05-19 03:54:45 +0000567 if (Error E = Result.takeError()) {
568 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
569 auto Err = IPE.get();
570 bool SkipWarning = false;
571 if (Err == instrprof_error::unknown_function) {
572 NumOfPGOMissing++;
573 SkipWarning = NoPGOWarnMissing;
574 } else if (Err == instrprof_error::hash_mismatch ||
575 Err == instrprof_error::malformed) {
576 NumOfPGOMismatch++;
577 SkipWarning = NoPGOWarnMismatch;
578 }
Rong Xuf430ae42015-12-09 18:08:16 +0000579
Vedant Kumar9152fd12016-05-19 03:54:45 +0000580 if (SkipWarning)
581 return;
582
583 std::string Msg = IPE.message() + std::string(" ") + F.getName().str();
584 Ctx.diagnose(
585 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
586 });
Rong Xuf430ae42015-12-09 18:08:16 +0000587 return false;
588 }
Rong Xu13b01dc2016-02-10 18:24:45 +0000589 ProfileRecord = std::move(Result.get());
590 std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
Rong Xuf430ae42015-12-09 18:08:16 +0000591
592 NumOfPGOFunc++;
593 DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
594 uint64_t ValueSum = 0;
595 for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
596 DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n");
597 ValueSum += CountFromProfile[I];
598 }
599
600 DEBUG(dbgs() << "SUM = " << ValueSum << "\n");
601
602 getBBInfo(nullptr).UnknownCountOutEdge = 2;
603 getBBInfo(nullptr).UnknownCountInEdge = 2;
604
605 setInstrumentedCounts(CountFromProfile);
Rong Xuf430ae42015-12-09 18:08:16 +0000606 return true;
607}
608
609// Populate the counters from instrumented BBs to all BBs.
610// In the end of this operation, all BBs should have a valid count value.
611void PGOUseFunc::populateCounters() {
612 // First set up Count variable for all BBs.
613 for (auto &E : FuncInfo.MST.AllEdges) {
614 if (E->Removed)
615 continue;
616
617 const BasicBlock *SrcBB = E->SrcBB;
618 const BasicBlock *DestBB = E->DestBB;
619 UseBBInfo &SrcInfo = getBBInfo(SrcBB);
620 UseBBInfo &DestInfo = getBBInfo(DestBB);
621 SrcInfo.OutEdges.push_back(E.get());
622 DestInfo.InEdges.push_back(E.get());
623 SrcInfo.UnknownCountOutEdge++;
624 DestInfo.UnknownCountInEdge++;
625
626 if (!E->CountValid)
627 continue;
628 DestInfo.UnknownCountInEdge--;
629 SrcInfo.UnknownCountOutEdge--;
630 }
631
632 bool Changes = true;
633 unsigned NumPasses = 0;
634 while (Changes) {
635 NumPasses++;
636 Changes = false;
637
638 // For efficient traversal, it's better to start from the end as most
639 // of the instrumented edges are at the end.
640 for (auto &BB : reverse(F)) {
641 UseBBInfo &Count = getBBInfo(&BB);
642 if (!Count.CountValid) {
643 if (Count.UnknownCountOutEdge == 0) {
644 Count.CountValue = sumEdgeCount(Count.OutEdges);
645 Count.CountValid = true;
646 Changes = true;
647 } else if (Count.UnknownCountInEdge == 0) {
648 Count.CountValue = sumEdgeCount(Count.InEdges);
649 Count.CountValid = true;
650 Changes = true;
651 }
652 }
653 if (Count.CountValid) {
654 if (Count.UnknownCountOutEdge == 1) {
655 uint64_t Total = Count.CountValue - sumEdgeCount(Count.OutEdges);
656 setEdgeCount(Count.OutEdges, Total);
657 Changes = true;
658 }
659 if (Count.UnknownCountInEdge == 1) {
660 uint64_t Total = Count.CountValue - sumEdgeCount(Count.InEdges);
661 setEdgeCount(Count.InEdges, Total);
662 Changes = true;
663 }
664 }
665 }
666 }
667
668 DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n");
Sean Silva2a730192016-05-28 03:02:50 +0000669
670 EntryCount = getBBInfo(&*F.begin()).CountValue;
671 MaxBBCount = 0;
Rong Xuf430ae42015-12-09 18:08:16 +0000672 for (auto &BB : F) {
Sean Silva2a730192016-05-28 03:02:50 +0000673 // Assert every BB has a valid counter.
Rong Xuf430ae42015-12-09 18:08:16 +0000674 assert(getBBInfo(&BB).CountValid && "BB count is not valid");
Sean Silva2a730192016-05-28 03:02:50 +0000675 MaxBBCount = std::max(MaxBBCount, getBBInfo(&BB).CountValue);
Rong Xuf430ae42015-12-09 18:08:16 +0000676 }
Rong Xuf430ae42015-12-09 18:08:16 +0000677
678 DEBUG(FuncInfo.dumpInfo("after reading profile."));
679}
680
681// Assign the scaled count values to the BB with multiple out edges.
682void PGOUseFunc::setBranchWeights() {
683 // Generate MD_prof metadata for every branch instruction.
684 DEBUG(dbgs() << "\nSetting branch weights.\n");
685 MDBuilder MDB(M->getContext());
686 for (auto &BB : F) {
687 TerminatorInst *TI = BB.getTerminator();
688 if (TI->getNumSuccessors() < 2)
689 continue;
690 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
691 continue;
692 if (getBBInfo(&BB).CountValue == 0)
693 continue;
694
695 // We have a non-zero Branch BB.
696 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
697 unsigned Size = BBCountInfo.OutEdges.size();
698 SmallVector<unsigned, 2> EdgeCounts(Size, 0);
699 uint64_t MaxCount = 0;
700 for (unsigned s = 0; s < Size; s++) {
701 const PGOUseEdge *E = BBCountInfo.OutEdges[s];
702 const BasicBlock *SrcBB = E->SrcBB;
703 const BasicBlock *DestBB = E->DestBB;
Eugene Zelenko6ac3f732016-01-26 18:48:36 +0000704 if (DestBB == nullptr)
Rong Xuf430ae42015-12-09 18:08:16 +0000705 continue;
706 unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
707 uint64_t EdgeCount = E->CountValue;
708 if (EdgeCount > MaxCount)
709 MaxCount = EdgeCount;
710 EdgeCounts[SuccNum] = EdgeCount;
711 }
712 assert(MaxCount > 0 && "Bad max count");
713 uint64_t Scale = calculateCountScale(MaxCount);
714 SmallVector<unsigned, 4> Weights;
715 for (const auto &ECI : EdgeCounts)
716 Weights.push_back(scaleBranchCount(ECI, Scale));
717
718 TI->setMetadata(llvm::LLVMContext::MD_prof,
719 MDB.createBranchWeights(Weights));
720 DEBUG(dbgs() << "Weight is: ";
721 for (const auto &W : Weights) { dbgs() << W << " "; }
722 dbgs() << "\n";);
723 }
724}
Rong Xu13b01dc2016-02-10 18:24:45 +0000725
726// Traverse all the indirect callsites and annotate the instructions.
727void PGOUseFunc::annotateIndirectCallSites() {
728 if (DisableValueProfiling)
729 return;
730
Rong Xu8e8fe852016-04-01 16:43:30 +0000731 // Create the PGOFuncName meta data.
Rong Xuf8f051c2016-04-22 21:00:17 +0000732 createPGOFuncNameMetadata(F, FuncInfo.FuncName);
Rong Xub5341662016-03-30 18:37:52 +0000733
Rong Xu13b01dc2016-02-10 18:24:45 +0000734 unsigned IndirectCallSiteIndex = 0;
Rong Xu0eb36032016-04-01 23:16:44 +0000735 auto IndirectCallSites = findIndirectCallSites(F);
Rong Xu9e926e82016-02-29 19:16:04 +0000736 unsigned NumValueSites =
Rong Xu13b01dc2016-02-10 18:24:45 +0000737 ProfileRecord.getNumValueSites(IPVK_IndirectCallTarget);
Rong Xu0eb36032016-04-01 23:16:44 +0000738 if (NumValueSites != IndirectCallSites.size()) {
Rong Xu13b01dc2016-02-10 18:24:45 +0000739 std::string Msg =
740 std::string("Inconsistent number of indirect call sites: ") +
Rong Xu9e926e82016-02-29 19:16:04 +0000741 F.getName().str();
Rong Xu13b01dc2016-02-10 18:24:45 +0000742 auto &Ctx = M->getContext();
743 Ctx.diagnose(
744 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
745 return;
746 }
747
Rong Xu0eb36032016-04-01 23:16:44 +0000748 for (auto &I : IndirectCallSites) {
Rong Xu13b01dc2016-02-10 18:24:45 +0000749 DEBUG(dbgs() << "Read one indirect call instrumentation: Index="
Rong Xu9e926e82016-02-29 19:16:04 +0000750 << IndirectCallSiteIndex << " out of " << NumValueSites
751 << "\n");
Rong Xu13b01dc2016-02-10 18:24:45 +0000752 annotateValueSite(*M, *I, ProfileRecord, IPVK_IndirectCallTarget,
Rong Xuecdc98f2016-03-04 22:08:44 +0000753 IndirectCallSiteIndex, MaxNumAnnotations);
Rong Xu13b01dc2016-02-10 18:24:45 +0000754 IndirectCallSiteIndex++;
755 }
756}
Rong Xuf430ae42015-12-09 18:08:16 +0000757} // end anonymous namespace
758
Rong Xu33c76c02016-02-10 17:18:30 +0000759// Create a COMDAT variable IR_LEVEL_PROF_VARNAME to make the runtime
760// aware this is an ir_level profile so it can set the version flag.
761static void createIRLevelProfileFlagVariable(Module &M) {
762 Type *IntTy64 = Type::getInt64Ty(M.getContext());
763 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
Rong Xu9e926e82016-02-29 19:16:04 +0000764 auto IRLevelVersionVariable = new GlobalVariable(
765 M, IntTy64, true, GlobalVariable::ExternalLinkage,
766 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)),
767 INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR));
Rong Xu33c76c02016-02-10 17:18:30 +0000768 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
769 Triple TT(M.getTargetTriple());
Xinliang David Li11c849c2016-05-27 16:22:03 +0000770 if (!TT.supportsCOMDAT())
Rong Xuca28a0a2016-05-11 00:31:59 +0000771 IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage);
Rong Xu33c76c02016-02-10 17:18:30 +0000772 else
Rong Xu9e926e82016-02-29 19:16:04 +0000773 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(
774 StringRef(INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR))));
Rong Xu33c76c02016-02-10 17:18:30 +0000775}
776
Xinliang David Li5ad7c822016-05-02 20:33:59 +0000777static bool InstrumentAllFunctions(
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000778 Module &M, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
779 function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) {
Rong Xu33c76c02016-02-10 17:18:30 +0000780 createIRLevelProfileFlagVariable(M);
Rong Xuf430ae42015-12-09 18:08:16 +0000781 for (auto &F : M) {
782 if (F.isDeclaration())
783 continue;
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000784 auto *BPI = LookupBPI(F);
785 auto *BFI = LookupBFI(F);
786 instrumentOneFunc(F, &M, BPI, BFI);
Rong Xuf430ae42015-12-09 18:08:16 +0000787 }
788 return true;
789}
790
Xinliang David Li8aebf442016-05-06 05:49:19 +0000791bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) {
Xinliang David Li5ad7c822016-05-02 20:33:59 +0000792 if (skipModule(M))
793 return false;
794
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000795 auto LookupBPI = [this](Function &F) {
796 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
Xinliang David Li5ad7c822016-05-02 20:33:59 +0000797 };
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000798 auto LookupBFI = [this](Function &F) {
799 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
Xinliang David Li5ad7c822016-05-02 20:33:59 +0000800 };
801 return InstrumentAllFunctions(M, LookupBPI, LookupBFI);
802}
803
Xinliang David Li8aebf442016-05-06 05:49:19 +0000804PreservedAnalyses PGOInstrumentationGen::run(Module &M,
805 AnalysisManager<Module> &AM) {
806
807 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000808 auto LookupBPI = [&FAM](Function &F) {
809 return &FAM.getResult<BranchProbabilityAnalysis>(F);
Xinliang David Li8aebf442016-05-06 05:49:19 +0000810 };
811
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000812 auto LookupBFI = [&FAM](Function &F) {
813 return &FAM.getResult<BlockFrequencyAnalysis>(F);
Xinliang David Li8aebf442016-05-06 05:49:19 +0000814 };
815
816 if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI))
817 return PreservedAnalyses::all();
818
819 return PreservedAnalyses::none();
820}
821
Rong Xuf430ae42015-12-09 18:08:16 +0000822static void setPGOCountOnFunc(PGOUseFunc &Func,
823 IndexedInstrProfReader *PGOReader) {
824 if (Func.readCounters(PGOReader)) {
825 Func.populateCounters();
826 Func.setBranchWeights();
Rong Xu13b01dc2016-02-10 18:24:45 +0000827 Func.annotateIndirectCallSites();
Rong Xuf430ae42015-12-09 18:08:16 +0000828 }
829}
830
Xinliang David Lida195582016-05-10 21:59:52 +0000831static bool annotateAllFunctions(
832 Module &M, StringRef ProfileFileName,
833 function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000834 function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) {
Rong Xuf430ae42015-12-09 18:08:16 +0000835 DEBUG(dbgs() << "Read in profile counters: ");
836 auto &Ctx = M.getContext();
837 // Read the counter array from file.
838 auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName);
Vedant Kumar9152fd12016-05-19 03:54:45 +0000839 if (Error E = ReaderOrErr.takeError()) {
840 handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) {
841 Ctx.diagnose(
842 DiagnosticInfoPGOProfile(ProfileFileName.data(), EI.message()));
843 });
Rong Xuf430ae42015-12-09 18:08:16 +0000844 return false;
845 }
846
Xinliang David Lida195582016-05-10 21:59:52 +0000847 std::unique_ptr<IndexedInstrProfReader> PGOReader =
848 std::move(ReaderOrErr.get());
Rong Xuf430ae42015-12-09 18:08:16 +0000849 if (!PGOReader) {
850 Ctx.diagnose(DiagnosticInfoPGOProfile(ProfileFileName.data(),
Xinliang David Lida195582016-05-10 21:59:52 +0000851 StringRef("Cannot get PGOReader")));
Rong Xuf430ae42015-12-09 18:08:16 +0000852 return false;
853 }
Rong Xu33c76c02016-02-10 17:18:30 +0000854 // TODO: might need to change the warning once the clang option is finalized.
855 if (!PGOReader->isIRLevelProfile()) {
856 Ctx.diagnose(DiagnosticInfoPGOProfile(
857 ProfileFileName.data(), "Not an IR level instrumentation profile"));
858 return false;
859 }
860
Rong Xu6090afd2016-03-28 17:08:56 +0000861 std::vector<Function *> HotFunctions;
862 std::vector<Function *> ColdFunctions;
Easwaran Raman5fe04a12016-05-26 22:57:11 +0000863 InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
Sean Silva2a730192016-05-28 03:02:50 +0000864 uint64_t ProgramMaxCount = PGOReader->getMaximumFunctionCount();
865 bool HasProgramMaxCount = ProgramMaxCount != 0;
Rong Xuf430ae42015-12-09 18:08:16 +0000866 for (auto &F : M) {
867 if (F.isDeclaration())
868 continue;
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000869 auto *BPI = LookupBPI(F);
870 auto *BFI = LookupBFI(F);
871 PGOUseFunc Func(F, &M, BPI, BFI);
Rong Xuf430ae42015-12-09 18:08:16 +0000872 setPGOCountOnFunc(Func, PGOReader.get());
Easwaran Raman5fe04a12016-05-26 22:57:11 +0000873 if (!Func.getProfileRecord().Counts.empty())
874 Builder.addRecord(Func.getProfileRecord());
Sean Silva2a730192016-05-28 03:02:50 +0000875
876 if (!HasProgramMaxCount)
877 continue;
878 // Set the hot/cold inline hints based on the count values.
879 // FIXME: This should be removed once the functionality in
880 // the inliner is implemented.
881 const BranchProbability HotFunctionThreshold(1, 100);
882 const BranchProbability ColdFunctionThreshold(2, 10000);
883 if (Func.EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount))
Rong Xu6090afd2016-03-28 17:08:56 +0000884 HotFunctions.push_back(&F);
Sean Silva2a730192016-05-28 03:02:50 +0000885 else if (Func.MaxBBCount <= ColdFunctionThreshold.scale(ProgramMaxCount))
886 ColdFunctions.push_back(&F);
Rong Xuf430ae42015-12-09 18:08:16 +0000887 }
Easwaran Raman5fe04a12016-05-26 22:57:11 +0000888 M.setProfileSummary(Builder.getSummary()->getMD(M.getContext()));
Sean Silva2a730192016-05-28 03:02:50 +0000889
Rong Xu6090afd2016-03-28 17:08:56 +0000890 // Set function hotness attribute from the profile.
Sean Silva2a730192016-05-28 03:02:50 +0000891 // We have to apply these attributes at the end because their presence
892 // can affect the BranchProbabilityInfo of any callers, resulting in an
893 // inconsistent MST between prof-gen and prof-use.
Rong Xu6090afd2016-03-28 17:08:56 +0000894 for (auto &F : HotFunctions) {
895 F->addFnAttr(llvm::Attribute::InlineHint);
896 DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()
897 << "\n");
898 }
899 for (auto &F : ColdFunctions) {
900 F->addFnAttr(llvm::Attribute::Cold);
901 DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n");
902 }
903
Rong Xuf430ae42015-12-09 18:08:16 +0000904 return true;
905}
Xinliang David Lid55827f2016-05-07 05:39:12 +0000906
Xinliang David Lida195582016-05-10 21:59:52 +0000907PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename)
Benjamin Kramer82de7d32016-05-27 14:27:24 +0000908 : ProfileFileName(std::move(Filename)) {
Xinliang David Lida195582016-05-10 21:59:52 +0000909 if (!PGOTestProfileFile.empty())
910 ProfileFileName = PGOTestProfileFile;
911}
912
913PreservedAnalyses PGOInstrumentationUse::run(Module &M,
914 AnalysisManager<Module> &AM) {
915
916 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
917 auto LookupBPI = [&FAM](Function &F) {
918 return &FAM.getResult<BranchProbabilityAnalysis>(F);
919 };
920
921 auto LookupBFI = [&FAM](Function &F) {
922 return &FAM.getResult<BlockFrequencyAnalysis>(F);
923 };
924
925 if (!annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI))
926 return PreservedAnalyses::all();
927
928 return PreservedAnalyses::none();
929}
930
Xinliang David Lid55827f2016-05-07 05:39:12 +0000931bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) {
932 if (skipModule(M))
933 return false;
934
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000935 auto LookupBPI = [this](Function &F) {
936 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
Xinliang David Lid55827f2016-05-07 05:39:12 +0000937 };
Xinliang David Lidfa21c32016-05-09 21:37:12 +0000938 auto LookupBFI = [this](Function &F) {
939 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
Xinliang David Lid55827f2016-05-07 05:39:12 +0000940 };
941
Xinliang David Lida195582016-05-10 21:59:52 +0000942 return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI);
Xinliang David Lid55827f2016-05-07 05:39:12 +0000943}