blob: 431c850ef81ea26342d5503c427bf36f92897622 [file] [log] [blame]
Justin Bogneref512b92014-01-06 22:27:43 +00001//===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Instrumentation-based profile-guided optimization
11//
12//===----------------------------------------------------------------------===//
13
Benjamin Kramer2f5db8b2014-08-13 16:25:19 +000014#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H
15#define LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H
Justin Bogneref512b92014-01-06 22:27:43 +000016
17#include "CGBuilder.h"
18#include "CodeGenModule.h"
19#include "CodeGenTypes.h"
20#include "clang/Frontend/CodeGenOptions.h"
Justin Bogneref512b92014-01-06 22:27:43 +000021#include "llvm/ADT/StringMap.h"
22#include "llvm/Support/MemoryBuffer.h"
Ahmed Charlesdfca6f92014-03-09 11:36:40 +000023#include <memory>
Justin Bogneref512b92014-01-06 22:27:43 +000024
25namespace clang {
26namespace CodeGen {
27class RegionCounter;
28
Justin Bogneref512b92014-01-06 22:27:43 +000029/// Per-function PGO state. This class should generally not be used directly,
30/// but instead through the CodeGenFunction and RegionCounter types.
31class CodeGenPGO {
32private:
33 CodeGenModule &CGM;
Justin Bogner111c6532014-12-02 23:15:30 +000034 std::string FuncName;
Justin Bogner970ac602014-12-08 19:04:51 +000035 llvm::GlobalVariable *FuncNameVar;
Justin Bogneref512b92014-01-06 22:27:43 +000036
37 unsigned NumRegionCounters;
Justin Bognerb4416f52014-03-18 21:58:06 +000038 uint64_t FunctionHash;
Duncan P. N. Exon Smith1b67cfd2014-03-26 19:26:05 +000039 std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap;
40 std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap;
Justin Bogner7f8cf5b2014-12-02 22:38:52 +000041 std::vector<uint64_t> RegionCounts;
Justin Bogneref512b92014-01-06 22:27:43 +000042 uint64_t CurrentRegionCount;
Alex Lorenzee024992014-08-04 18:41:51 +000043 /// \brief A flag that is set to true when this function doesn't need
44 /// to have coverage mapping data.
45 bool SkipCoverageMapping;
Justin Bogneref512b92014-01-06 22:27:43 +000046
47public:
48 CodeGenPGO(CodeGenModule &CGM)
Justin Bogner970ac602014-12-08 19:04:51 +000049 : CGM(CGM), NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0),
Alex Lorenzee024992014-08-04 18:41:51 +000050 SkipCoverageMapping(false) {}
Justin Bogneref512b92014-01-06 22:27:43 +000051
52 /// Whether or not we have PGO region data for the current function. This is
53 /// false both when we have no data at all and when our data has been
54 /// discarded.
Justin Bogner7f8cf5b2014-12-02 22:38:52 +000055 bool haveRegionCounts() const { return !RegionCounts.empty(); }
Justin Bogneref512b92014-01-06 22:27:43 +000056
57 /// Return the counter value of the current region.
58 uint64_t getCurrentRegionCount() const { return CurrentRegionCount; }
Bob Wilsonbf854f02014-02-17 19:21:09 +000059
Justin Bogneref512b92014-01-06 22:27:43 +000060 /// Set the counter value for the current region. This is used to keep track
61 /// of changes to the most recent counter from control flow and non-local
62 /// exits.
63 void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; }
Bob Wilsonbf854f02014-02-17 19:21:09 +000064
Justin Bogner06bd6d02014-01-13 21:24:18 +000065 /// Indicate that the current region is never reached, and thus should have a
66 /// counter value of zero. This is important so that subsequent regions can
67 /// correctly track their parent counts.
68 void setCurrentRegionUnreachable() { setCurrentRegionCount(0); }
Justin Bogneref512b92014-01-06 22:27:43 +000069
Bob Wilsonbf854f02014-02-17 19:21:09 +000070 /// Check if an execution count is known for a given statement. If so, return
71 /// true and put the value in Count; else return false.
72 bool getStmtCount(const Stmt *S, uint64_t &Count) {
73 if (!StmtCountMap)
74 return false;
75 llvm::DenseMap<const Stmt*, uint64_t>::const_iterator
76 I = StmtCountMap->find(S);
77 if (I == StmtCountMap->end())
78 return false;
79 Count = I->second;
80 return true;
81 }
82
83 /// If the execution count for the current statement is known, record that
84 /// as the current count.
85 void setCurrentStmt(const Stmt *S) {
86 uint64_t Count;
87 if (getStmtCount(S, Count))
88 setCurrentRegionCount(Count);
89 }
90
Justin Bogneref512b92014-01-06 22:27:43 +000091 /// Calculate branch weights appropriate for PGO data
92 llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount);
93 llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights);
Bob Wilsonbf854f02014-02-17 19:21:09 +000094 llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt);
Justin Bogneref512b92014-01-06 22:27:43 +000095
Alex Lorenzee024992014-08-04 18:41:51 +000096 /// Check if we need to emit coverage mapping for a given declaration
97 void checkGlobalDecl(GlobalDecl GD);
Justin Bogneref512b92014-01-06 22:27:43 +000098 /// Assign counters to regions and configure them for PGO of a given
99 /// function. Does nothing if instrumentation is not enabled and either
100 /// generates global variables or associates PGO data with each of the
101 /// counters depending on whether we are generating or using instrumentation.
Bob Wilsonda1ebed2014-03-06 04:55:41 +0000102 void assignRegionCounters(const Decl *D, llvm::Function *Fn);
Alex Lorenzee024992014-08-04 18:41:51 +0000103 /// Emit a coverage mapping range with a counter zero
104 /// for an unused declaration.
105 void emitEmptyCounterMapping(const Decl *D, StringRef FuncName,
106 llvm::GlobalValue::LinkageTypes Linkage);
Justin Bogneref512b92014-01-06 22:27:43 +0000107private:
Bob Wilsonda1ebed2014-03-06 04:55:41 +0000108 void setFuncName(llvm::Function *Fn);
Alex Lorenzee024992014-08-04 18:41:51 +0000109 void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage);
Justin Bogner970ac602014-12-08 19:04:51 +0000110 void createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage);
Justin Bogneref512b92014-01-06 22:27:43 +0000111 void mapRegionCounters(const Decl *D);
Bob Wilsonbf854f02014-02-17 19:21:09 +0000112 void computeRegionCounts(const Decl *D);
Justin Bogner837a6f62014-04-18 21:52:00 +0000113 void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
114 llvm::Function *Fn);
Justin Bogner40b8ba12014-06-26 01:45:07 +0000115 void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
116 bool IsInMainFile);
Justin Bogneref512b92014-01-06 22:27:43 +0000117 void emitCounterVariables();
Alex Lorenzee024992014-08-04 18:41:51 +0000118 void emitCounterRegionMapping(const Decl *D);
Justin Bogneref512b92014-01-06 22:27:43 +0000119
120 /// Emit code to increment the counter at the given index
121 void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter);
122
123 /// Return the region counter for the given statement. This should only be
124 /// called on statements that have a dedicated counter.
125 unsigned getRegionCounter(const Stmt *S) {
Craig Topper8a13c412014-05-21 05:09:00 +0000126 if (!RegionCounterMap)
Justin Bogneref512b92014-01-06 22:27:43 +0000127 return 0;
128 return (*RegionCounterMap)[S];
129 }
130
131 /// Return the region count for the counter at the given index.
132 uint64_t getRegionCount(unsigned Counter) {
133 if (!haveRegionCounts())
134 return 0;
Justin Bogner7f8cf5b2014-12-02 22:38:52 +0000135 return RegionCounts[Counter];
Justin Bogneref512b92014-01-06 22:27:43 +0000136 }
137
138 friend class RegionCounter;
139};
140
141/// A counter for a particular region. This is the primary interface through
142/// which clients manage PGO counters and their values.
143class RegionCounter {
144 CodeGenPGO *PGO;
145 unsigned Counter;
146 uint64_t Count;
147 uint64_t ParentCount;
148 uint64_t RegionCount;
149 int64_t Adjust;
150
151 RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex)
152 : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)),
153 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
154
155public:
156 RegionCounter(CodeGenPGO &PGO, const Stmt *S)
157 : PGO(&PGO), Counter(PGO.getRegionCounter(S)),
158 Count(PGO.getRegionCount(Counter)),
159 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
160
161 /// Get the value of the counter. In most cases this is the number of times
162 /// the region of the counter was entered, but for switch labels it's the
163 /// number of direct jumps to that label.
164 uint64_t getCount() const { return Count; }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000165
Justin Bogneref512b92014-01-06 22:27:43 +0000166 /// Get the value of the counter with adjustments applied. Adjustments occur
Bob Wilsona7b16e02014-02-17 19:21:03 +0000167 /// when control enters or leaves the region abnormally; i.e., if there is a
Justin Bogneref512b92014-01-06 22:27:43 +0000168 /// jump to a label within the region, or if the function can return from
169 /// within the region. The adjusted count, then, is the value of the counter
170 /// at the end of the region.
171 uint64_t getAdjustedCount() const {
Justin Bogneref512b92014-01-06 22:27:43 +0000172 return Count + Adjust;
173 }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000174
Bob Wilsona7b16e02014-02-17 19:21:03 +0000175 /// Get the value of the counter in this region's parent, i.e., the region
176 /// that was active when this region began. This is useful for deriving
177 /// counts in implicitly counted regions, like the false case of a condition
178 /// or the normal exits of a loop.
Justin Bogneref512b92014-01-06 22:27:43 +0000179 uint64_t getParentCount() const { return ParentCount; }
180
Justin Bogneref512b92014-01-06 22:27:43 +0000181 /// Activate the counter by emitting an increment and starting to track
182 /// adjustments. If AddIncomingFallThrough is true, the current region count
183 /// will be added to the counter for the purposes of tracking the region.
184 void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) {
Bob Wilsonbf854f02014-02-17 19:21:09 +0000185 beginRegion(AddIncomingFallThrough);
186 PGO->emitCounterIncrement(Builder, Counter);
187 }
188 void beginRegion(bool AddIncomingFallThrough=false) {
Justin Bogneref512b92014-01-06 22:27:43 +0000189 RegionCount = Count;
190 if (AddIncomingFallThrough)
191 RegionCount += PGO->getCurrentRegionCount();
192 PGO->setCurrentRegionCount(RegionCount);
Justin Bogneref512b92014-01-06 22:27:43 +0000193 }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000194
Justin Bogneref512b92014-01-06 22:27:43 +0000195 /// For counters on boolean branches, begins tracking adjustments for the
196 /// uncounted path.
197 void beginElseRegion() {
198 RegionCount = ParentCount - Count;
199 PGO->setCurrentRegionCount(RegionCount);
200 }
201
Bob Wilsonbf854f02014-02-17 19:21:09 +0000202 /// Reset the current region count.
203 void setCurrentRegionCount(uint64_t CurrentCount) {
204 RegionCount = CurrentCount;
205 PGO->setCurrentRegionCount(RegionCount);
206 }
207
Justin Bogner0718a3a2014-01-13 21:24:22 +0000208 /// Adjust for non-local control flow after emitting a subexpression or
209 /// substatement. This must be called to account for constructs such as gotos,
210 /// labels, and returns, so that we can ensure that our region's count is
211 /// correct in the code that follows.
212 void adjustForControlFlow() {
Justin Bogneref512b92014-01-06 22:27:43 +0000213 Adjust += PGO->getCurrentRegionCount() - RegionCount;
Bob Wilsonbf854f02014-02-17 19:21:09 +0000214 // Reset the region count in case this is called again later.
215 RegionCount = PGO->getCurrentRegionCount();
Justin Bogneref512b92014-01-06 22:27:43 +0000216 }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000217
218 /// Commit all adjustments to the current region. If the region is a loop,
219 /// the LoopAdjust value should be the count of all the breaks and continues
220 /// from the loop, to compensate for those counts being deducted from the
221 /// adjustments for the body of the loop.
222 void applyAdjustmentsToRegion(uint64_t LoopAdjust) {
223 PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust);
Justin Bogneref512b92014-01-06 22:27:43 +0000224 }
225};
226
227} // end namespace CodeGen
228} // end namespace clang
229
230#endif