blob: 0fc570ad9cda4e9d8873126b16d10c18cb7780dc [file] [log] [blame]
Justin Bogneref512b92014-01-06 22:27:43 +00001//===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Instrumentation-based profile-guided optimization
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef CLANG_CODEGEN_CODEGENPGO_H
15#define CLANG_CODEGEN_CODEGENPGO_H
16
17#include "CGBuilder.h"
18#include "CodeGenModule.h"
19#include "CodeGenTypes.h"
20#include "clang/Frontend/CodeGenOptions.h"
21#include "llvm/ADT/OwningPtr.h"
22#include "llvm/ADT/StringMap.h"
23#include "llvm/Support/MemoryBuffer.h"
24
25namespace clang {
26namespace CodeGen {
27class RegionCounter;
28
29/// The raw counter data from an instrumented PGO binary
30class PGOProfileData {
31private:
32 /// The PGO data
33 llvm::OwningPtr<llvm::MemoryBuffer> DataBuffer;
34 /// Offsets into DataBuffer for each function's counters
35 llvm::StringMap<unsigned> DataOffsets;
Manman Ren67a28132014-02-05 20:40:15 +000036 /// Execution counts for each function.
37 llvm::StringMap<uint64_t> FunctionCounts;
38 /// The maximal execution count among all functions.
39 uint64_t MaxFunctionCount;
Justin Bogneref512b92014-01-06 22:27:43 +000040 CodeGenModule &CGM;
41public:
42 PGOProfileData(CodeGenModule &CGM, std::string Path);
43 /// Fill Counts with the profile data for the given function name. Returns
44 /// false on success.
Bob Wilsond0b78242014-03-06 04:55:37 +000045 bool getFunctionCounts(StringRef FuncName, std::vector<uint64_t> &Counts);
Manman Ren67a28132014-02-05 20:40:15 +000046 /// Return true if a function is hot. If we know nothing about the function,
47 /// return false.
Bob Wilsond0b78242014-03-06 04:55:37 +000048 bool isHotFunction(StringRef FuncName);
Manman Ren67a28132014-02-05 20:40:15 +000049 /// Return true if a function is cold. If we know nothing about the function,
50 /// return false.
Bob Wilsond0b78242014-03-06 04:55:37 +000051 bool isColdFunction(StringRef FuncName);
Justin Bogneref512b92014-01-06 22:27:43 +000052};
53
54/// Per-function PGO state. This class should generally not be used directly,
55/// but instead through the CodeGenFunction and RegionCounter types.
56class CodeGenPGO {
57private:
58 CodeGenModule &CGM;
Bob Wilsonda1ebed2014-03-06 04:55:41 +000059 std::string *FuncName;
Justin Bogneref512b92014-01-06 22:27:43 +000060
61 unsigned NumRegionCounters;
62 llvm::GlobalVariable *RegionCounters;
63 llvm::DenseMap<const Stmt*, unsigned> *RegionCounterMap;
Bob Wilsonbf854f02014-02-17 19:21:09 +000064 llvm::DenseMap<const Stmt*, uint64_t> *StmtCountMap;
Justin Bogneref512b92014-01-06 22:27:43 +000065 std::vector<uint64_t> *RegionCounts;
66 uint64_t CurrentRegionCount;
67
68public:
69 CodeGenPGO(CodeGenModule &CGM)
Bob Wilsonda1ebed2014-03-06 04:55:41 +000070 : CGM(CGM), FuncName(0), NumRegionCounters(0), RegionCounters(0),
71 RegionCounterMap(0), StmtCountMap(0), RegionCounts(0),
72 CurrentRegionCount(0) {}
73 ~CodeGenPGO() {
74 if (FuncName) delete FuncName;
75 }
Justin Bogneref512b92014-01-06 22:27:43 +000076
77 /// Whether or not we have PGO region data for the current function. This is
78 /// false both when we have no data at all and when our data has been
79 /// discarded.
80 bool haveRegionCounts() const { return RegionCounts != 0; }
81
Bob Wilsonda1ebed2014-03-06 04:55:41 +000082 /// Get the string used to identify this function in the profile data.
83 /// For functions with local linkage, this includes the main file name.
84 const StringRef getFuncName() const { return StringRef(*FuncName); }
85
Justin Bogneref512b92014-01-06 22:27:43 +000086 /// Return the counter value of the current region.
87 uint64_t getCurrentRegionCount() const { return CurrentRegionCount; }
Bob Wilsonbf854f02014-02-17 19:21:09 +000088
Justin Bogneref512b92014-01-06 22:27:43 +000089 /// Set the counter value for the current region. This is used to keep track
90 /// of changes to the most recent counter from control flow and non-local
91 /// exits.
92 void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; }
Bob Wilsonbf854f02014-02-17 19:21:09 +000093
Justin Bogner06bd6d02014-01-13 21:24:18 +000094 /// Indicate that the current region is never reached, and thus should have a
95 /// counter value of zero. This is important so that subsequent regions can
96 /// correctly track their parent counts.
97 void setCurrentRegionUnreachable() { setCurrentRegionCount(0); }
Justin Bogneref512b92014-01-06 22:27:43 +000098
Bob Wilsonbf854f02014-02-17 19:21:09 +000099 /// Check if an execution count is known for a given statement. If so, return
100 /// true and put the value in Count; else return false.
101 bool getStmtCount(const Stmt *S, uint64_t &Count) {
102 if (!StmtCountMap)
103 return false;
104 llvm::DenseMap<const Stmt*, uint64_t>::const_iterator
105 I = StmtCountMap->find(S);
106 if (I == StmtCountMap->end())
107 return false;
108 Count = I->second;
109 return true;
110 }
111
112 /// If the execution count for the current statement is known, record that
113 /// as the current count.
114 void setCurrentStmt(const Stmt *S) {
115 uint64_t Count;
116 if (getStmtCount(S, Count))
117 setCurrentRegionCount(Count);
118 }
119
Justin Bogneref512b92014-01-06 22:27:43 +0000120 /// Calculate branch weights appropriate for PGO data
121 llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount);
122 llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights);
Bob Wilsonbf854f02014-02-17 19:21:09 +0000123 llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt);
Justin Bogneref512b92014-01-06 22:27:43 +0000124
125 /// Assign counters to regions and configure them for PGO of a given
126 /// function. Does nothing if instrumentation is not enabled and either
127 /// generates global variables or associates PGO data with each of the
128 /// counters depending on whether we are generating or using instrumentation.
Bob Wilsonda1ebed2014-03-06 04:55:41 +0000129 void assignRegionCounters(const Decl *D, llvm::Function *Fn);
Justin Bogneref512b92014-01-06 22:27:43 +0000130 /// Emit code to write counts for a given function to disk, if necessary.
Bob Wilsonda1ebed2014-03-06 04:55:41 +0000131 void emitWriteoutFunction();
Justin Bogneref512b92014-01-06 22:27:43 +0000132 /// Clean up region counter state. Must be called if assignRegionCounters is
133 /// used.
134 void destroyRegionCounters();
135 /// Emit the logic to register region counter write out functions. Returns a
136 /// function that implements this logic.
137 static llvm::Function *emitInitialization(CodeGenModule &CGM);
138
139private:
Bob Wilsonda1ebed2014-03-06 04:55:41 +0000140 void setFuncName(llvm::Function *Fn);
Justin Bogneref512b92014-01-06 22:27:43 +0000141 void mapRegionCounters(const Decl *D);
Bob Wilsonbf854f02014-02-17 19:21:09 +0000142 void computeRegionCounts(const Decl *D);
Bob Wilsonda1ebed2014-03-06 04:55:41 +0000143 void loadRegionCounts(PGOProfileData *PGOData);
Justin Bogneref512b92014-01-06 22:27:43 +0000144 void emitCounterVariables();
145
146 /// Emit code to increment the counter at the given index
147 void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter);
148
149 /// Return the region counter for the given statement. This should only be
150 /// called on statements that have a dedicated counter.
151 unsigned getRegionCounter(const Stmt *S) {
152 if (RegionCounterMap == 0)
153 return 0;
154 return (*RegionCounterMap)[S];
155 }
156
157 /// Return the region count for the counter at the given index.
158 uint64_t getRegionCount(unsigned Counter) {
159 if (!haveRegionCounts())
160 return 0;
161 return (*RegionCounts)[Counter];
162 }
163
164 friend class RegionCounter;
165};
166
167/// A counter for a particular region. This is the primary interface through
168/// which clients manage PGO counters and their values.
169class RegionCounter {
170 CodeGenPGO *PGO;
171 unsigned Counter;
172 uint64_t Count;
173 uint64_t ParentCount;
174 uint64_t RegionCount;
175 int64_t Adjust;
176
177 RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex)
178 : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)),
179 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
180
181public:
182 RegionCounter(CodeGenPGO &PGO, const Stmt *S)
183 : PGO(&PGO), Counter(PGO.getRegionCounter(S)),
184 Count(PGO.getRegionCount(Counter)),
185 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
186
187 /// Get the value of the counter. In most cases this is the number of times
188 /// the region of the counter was entered, but for switch labels it's the
189 /// number of direct jumps to that label.
190 uint64_t getCount() const { return Count; }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000191
Justin Bogneref512b92014-01-06 22:27:43 +0000192 /// Get the value of the counter with adjustments applied. Adjustments occur
Bob Wilsona7b16e02014-02-17 19:21:03 +0000193 /// when control enters or leaves the region abnormally; i.e., if there is a
Justin Bogneref512b92014-01-06 22:27:43 +0000194 /// jump to a label within the region, or if the function can return from
195 /// within the region. The adjusted count, then, is the value of the counter
196 /// at the end of the region.
197 uint64_t getAdjustedCount() const {
Justin Bogneref512b92014-01-06 22:27:43 +0000198 return Count + Adjust;
199 }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000200
Bob Wilsona7b16e02014-02-17 19:21:03 +0000201 /// Get the value of the counter in this region's parent, i.e., the region
202 /// that was active when this region began. This is useful for deriving
203 /// counts in implicitly counted regions, like the false case of a condition
204 /// or the normal exits of a loop.
Justin Bogneref512b92014-01-06 22:27:43 +0000205 uint64_t getParentCount() const { return ParentCount; }
206
Justin Bogneref512b92014-01-06 22:27:43 +0000207 /// Activate the counter by emitting an increment and starting to track
208 /// adjustments. If AddIncomingFallThrough is true, the current region count
209 /// will be added to the counter for the purposes of tracking the region.
210 void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) {
Bob Wilsonbf854f02014-02-17 19:21:09 +0000211 beginRegion(AddIncomingFallThrough);
212 PGO->emitCounterIncrement(Builder, Counter);
213 }
214 void beginRegion(bool AddIncomingFallThrough=false) {
Justin Bogneref512b92014-01-06 22:27:43 +0000215 RegionCount = Count;
216 if (AddIncomingFallThrough)
217 RegionCount += PGO->getCurrentRegionCount();
218 PGO->setCurrentRegionCount(RegionCount);
Justin Bogneref512b92014-01-06 22:27:43 +0000219 }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000220
Justin Bogneref512b92014-01-06 22:27:43 +0000221 /// For counters on boolean branches, begins tracking adjustments for the
222 /// uncounted path.
223 void beginElseRegion() {
224 RegionCount = ParentCount - Count;
225 PGO->setCurrentRegionCount(RegionCount);
226 }
227
Bob Wilsonbf854f02014-02-17 19:21:09 +0000228 /// Reset the current region count.
229 void setCurrentRegionCount(uint64_t CurrentCount) {
230 RegionCount = CurrentCount;
231 PGO->setCurrentRegionCount(RegionCount);
232 }
233
Justin Bogner0718a3a2014-01-13 21:24:22 +0000234 /// Adjust for non-local control flow after emitting a subexpression or
235 /// substatement. This must be called to account for constructs such as gotos,
236 /// labels, and returns, so that we can ensure that our region's count is
237 /// correct in the code that follows.
238 void adjustForControlFlow() {
Justin Bogneref512b92014-01-06 22:27:43 +0000239 Adjust += PGO->getCurrentRegionCount() - RegionCount;
Bob Wilsonbf854f02014-02-17 19:21:09 +0000240 // Reset the region count in case this is called again later.
241 RegionCount = PGO->getCurrentRegionCount();
Justin Bogneref512b92014-01-06 22:27:43 +0000242 }
Bob Wilsonbf854f02014-02-17 19:21:09 +0000243
244 /// Commit all adjustments to the current region. If the region is a loop,
245 /// the LoopAdjust value should be the count of all the breaks and continues
246 /// from the loop, to compensate for those counts being deducted from the
247 /// adjustments for the body of the loop.
248 void applyAdjustmentsToRegion(uint64_t LoopAdjust) {
249 PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust);
Justin Bogneref512b92014-01-06 22:27:43 +0000250 }
251};
252
253} // end namespace CodeGen
254} // end namespace clang
255
256#endif