Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 1 | //===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // Instrumentation-based profile-guided optimization |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef CLANG_CODEGEN_CODEGENPGO_H |
| 15 | #define CLANG_CODEGEN_CODEGENPGO_H |
| 16 | |
| 17 | #include "CGBuilder.h" |
| 18 | #include "CodeGenModule.h" |
| 19 | #include "CodeGenTypes.h" |
| 20 | #include "clang/Frontend/CodeGenOptions.h" |
| 21 | #include "llvm/ADT/OwningPtr.h" |
| 22 | #include "llvm/ADT/StringMap.h" |
| 23 | #include "llvm/Support/MemoryBuffer.h" |
| 24 | |
| 25 | namespace clang { |
| 26 | namespace CodeGen { |
| 27 | class RegionCounter; |
| 28 | |
| 29 | /// The raw counter data from an instrumented PGO binary |
| 30 | class PGOProfileData { |
| 31 | private: |
| 32 | /// The PGO data |
| 33 | llvm::OwningPtr<llvm::MemoryBuffer> DataBuffer; |
| 34 | /// Offsets into DataBuffer for each function's counters |
| 35 | llvm::StringMap<unsigned> DataOffsets; |
Manman Ren | 67a2813 | 2014-02-05 20:40:15 +0000 | [diff] [blame] | 36 | /// Execution counts for each function. |
| 37 | llvm::StringMap<uint64_t> FunctionCounts; |
| 38 | /// The maximal execution count among all functions. |
| 39 | uint64_t MaxFunctionCount; |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 40 | CodeGenModule &CGM; |
| 41 | public: |
| 42 | PGOProfileData(CodeGenModule &CGM, std::string Path); |
| 43 | /// Fill Counts with the profile data for the given function name. Returns |
| 44 | /// false on success. |
Bob Wilson | d0b7824 | 2014-03-06 04:55:37 +0000 | [diff] [blame] | 45 | bool getFunctionCounts(StringRef FuncName, std::vector<uint64_t> &Counts); |
Manman Ren | 67a2813 | 2014-02-05 20:40:15 +0000 | [diff] [blame] | 46 | /// Return true if a function is hot. If we know nothing about the function, |
| 47 | /// return false. |
Bob Wilson | d0b7824 | 2014-03-06 04:55:37 +0000 | [diff] [blame] | 48 | bool isHotFunction(StringRef FuncName); |
Manman Ren | 67a2813 | 2014-02-05 20:40:15 +0000 | [diff] [blame] | 49 | /// Return true if a function is cold. If we know nothing about the function, |
| 50 | /// return false. |
Bob Wilson | d0b7824 | 2014-03-06 04:55:37 +0000 | [diff] [blame] | 51 | bool isColdFunction(StringRef FuncName); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 52 | }; |
| 53 | |
| 54 | /// Per-function PGO state. This class should generally not be used directly, |
| 55 | /// but instead through the CodeGenFunction and RegionCounter types. |
| 56 | class CodeGenPGO { |
| 57 | private: |
| 58 | CodeGenModule &CGM; |
Bob Wilson | da1ebed | 2014-03-06 04:55:41 +0000 | [diff] [blame^] | 59 | std::string *FuncName; |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 60 | |
| 61 | unsigned NumRegionCounters; |
| 62 | llvm::GlobalVariable *RegionCounters; |
| 63 | llvm::DenseMap<const Stmt*, unsigned> *RegionCounterMap; |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 64 | llvm::DenseMap<const Stmt*, uint64_t> *StmtCountMap; |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 65 | std::vector<uint64_t> *RegionCounts; |
| 66 | uint64_t CurrentRegionCount; |
| 67 | |
| 68 | public: |
| 69 | CodeGenPGO(CodeGenModule &CGM) |
Bob Wilson | da1ebed | 2014-03-06 04:55:41 +0000 | [diff] [blame^] | 70 | : CGM(CGM), FuncName(0), NumRegionCounters(0), RegionCounters(0), |
| 71 | RegionCounterMap(0), StmtCountMap(0), RegionCounts(0), |
| 72 | CurrentRegionCount(0) {} |
| 73 | ~CodeGenPGO() { |
| 74 | if (FuncName) delete FuncName; |
| 75 | } |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 76 | |
| 77 | /// Whether or not we have PGO region data for the current function. This is |
| 78 | /// false both when we have no data at all and when our data has been |
| 79 | /// discarded. |
| 80 | bool haveRegionCounts() const { return RegionCounts != 0; } |
| 81 | |
Bob Wilson | da1ebed | 2014-03-06 04:55:41 +0000 | [diff] [blame^] | 82 | /// Get the string used to identify this function in the profile data. |
| 83 | /// For functions with local linkage, this includes the main file name. |
| 84 | const StringRef getFuncName() const { return StringRef(*FuncName); } |
| 85 | |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 86 | /// Return the counter value of the current region. |
| 87 | uint64_t getCurrentRegionCount() const { return CurrentRegionCount; } |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 88 | |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 89 | /// Set the counter value for the current region. This is used to keep track |
| 90 | /// of changes to the most recent counter from control flow and non-local |
| 91 | /// exits. |
| 92 | void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 93 | |
Justin Bogner | 06bd6d0 | 2014-01-13 21:24:18 +0000 | [diff] [blame] | 94 | /// Indicate that the current region is never reached, and thus should have a |
| 95 | /// counter value of zero. This is important so that subsequent regions can |
| 96 | /// correctly track their parent counts. |
| 97 | void setCurrentRegionUnreachable() { setCurrentRegionCount(0); } |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 98 | |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 99 | /// Check if an execution count is known for a given statement. If so, return |
| 100 | /// true and put the value in Count; else return false. |
| 101 | bool getStmtCount(const Stmt *S, uint64_t &Count) { |
| 102 | if (!StmtCountMap) |
| 103 | return false; |
| 104 | llvm::DenseMap<const Stmt*, uint64_t>::const_iterator |
| 105 | I = StmtCountMap->find(S); |
| 106 | if (I == StmtCountMap->end()) |
| 107 | return false; |
| 108 | Count = I->second; |
| 109 | return true; |
| 110 | } |
| 111 | |
| 112 | /// If the execution count for the current statement is known, record that |
| 113 | /// as the current count. |
| 114 | void setCurrentStmt(const Stmt *S) { |
| 115 | uint64_t Count; |
| 116 | if (getStmtCount(S, Count)) |
| 117 | setCurrentRegionCount(Count); |
| 118 | } |
| 119 | |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 120 | /// Calculate branch weights appropriate for PGO data |
| 121 | llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount); |
| 122 | llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights); |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 123 | llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 124 | |
| 125 | /// Assign counters to regions and configure them for PGO of a given |
| 126 | /// function. Does nothing if instrumentation is not enabled and either |
| 127 | /// generates global variables or associates PGO data with each of the |
| 128 | /// counters depending on whether we are generating or using instrumentation. |
Bob Wilson | da1ebed | 2014-03-06 04:55:41 +0000 | [diff] [blame^] | 129 | void assignRegionCounters(const Decl *D, llvm::Function *Fn); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 130 | /// Emit code to write counts for a given function to disk, if necessary. |
Bob Wilson | da1ebed | 2014-03-06 04:55:41 +0000 | [diff] [blame^] | 131 | void emitWriteoutFunction(); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 132 | /// Clean up region counter state. Must be called if assignRegionCounters is |
| 133 | /// used. |
| 134 | void destroyRegionCounters(); |
| 135 | /// Emit the logic to register region counter write out functions. Returns a |
| 136 | /// function that implements this logic. |
| 137 | static llvm::Function *emitInitialization(CodeGenModule &CGM); |
| 138 | |
| 139 | private: |
Bob Wilson | da1ebed | 2014-03-06 04:55:41 +0000 | [diff] [blame^] | 140 | void setFuncName(llvm::Function *Fn); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 141 | void mapRegionCounters(const Decl *D); |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 142 | void computeRegionCounts(const Decl *D); |
Bob Wilson | da1ebed | 2014-03-06 04:55:41 +0000 | [diff] [blame^] | 143 | void loadRegionCounts(PGOProfileData *PGOData); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 144 | void emitCounterVariables(); |
| 145 | |
| 146 | /// Emit code to increment the counter at the given index |
| 147 | void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter); |
| 148 | |
| 149 | /// Return the region counter for the given statement. This should only be |
| 150 | /// called on statements that have a dedicated counter. |
| 151 | unsigned getRegionCounter(const Stmt *S) { |
| 152 | if (RegionCounterMap == 0) |
| 153 | return 0; |
| 154 | return (*RegionCounterMap)[S]; |
| 155 | } |
| 156 | |
| 157 | /// Return the region count for the counter at the given index. |
| 158 | uint64_t getRegionCount(unsigned Counter) { |
| 159 | if (!haveRegionCounts()) |
| 160 | return 0; |
| 161 | return (*RegionCounts)[Counter]; |
| 162 | } |
| 163 | |
| 164 | friend class RegionCounter; |
| 165 | }; |
| 166 | |
| 167 | /// A counter for a particular region. This is the primary interface through |
| 168 | /// which clients manage PGO counters and their values. |
| 169 | class RegionCounter { |
| 170 | CodeGenPGO *PGO; |
| 171 | unsigned Counter; |
| 172 | uint64_t Count; |
| 173 | uint64_t ParentCount; |
| 174 | uint64_t RegionCount; |
| 175 | int64_t Adjust; |
| 176 | |
| 177 | RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex) |
| 178 | : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)), |
| 179 | ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} |
| 180 | |
| 181 | public: |
| 182 | RegionCounter(CodeGenPGO &PGO, const Stmt *S) |
| 183 | : PGO(&PGO), Counter(PGO.getRegionCounter(S)), |
| 184 | Count(PGO.getRegionCount(Counter)), |
| 185 | ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} |
| 186 | |
| 187 | /// Get the value of the counter. In most cases this is the number of times |
| 188 | /// the region of the counter was entered, but for switch labels it's the |
| 189 | /// number of direct jumps to that label. |
| 190 | uint64_t getCount() const { return Count; } |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 191 | |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 192 | /// Get the value of the counter with adjustments applied. Adjustments occur |
Bob Wilson | a7b16e0 | 2014-02-17 19:21:03 +0000 | [diff] [blame] | 193 | /// when control enters or leaves the region abnormally; i.e., if there is a |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 194 | /// jump to a label within the region, or if the function can return from |
| 195 | /// within the region. The adjusted count, then, is the value of the counter |
| 196 | /// at the end of the region. |
| 197 | uint64_t getAdjustedCount() const { |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 198 | return Count + Adjust; |
| 199 | } |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 200 | |
Bob Wilson | a7b16e0 | 2014-02-17 19:21:03 +0000 | [diff] [blame] | 201 | /// Get the value of the counter in this region's parent, i.e., the region |
| 202 | /// that was active when this region began. This is useful for deriving |
| 203 | /// counts in implicitly counted regions, like the false case of a condition |
| 204 | /// or the normal exits of a loop. |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 205 | uint64_t getParentCount() const { return ParentCount; } |
| 206 | |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 207 | /// Activate the counter by emitting an increment and starting to track |
| 208 | /// adjustments. If AddIncomingFallThrough is true, the current region count |
| 209 | /// will be added to the counter for the purposes of tracking the region. |
| 210 | void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) { |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 211 | beginRegion(AddIncomingFallThrough); |
| 212 | PGO->emitCounterIncrement(Builder, Counter); |
| 213 | } |
| 214 | void beginRegion(bool AddIncomingFallThrough=false) { |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 215 | RegionCount = Count; |
| 216 | if (AddIncomingFallThrough) |
| 217 | RegionCount += PGO->getCurrentRegionCount(); |
| 218 | PGO->setCurrentRegionCount(RegionCount); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 219 | } |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 220 | |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 221 | /// For counters on boolean branches, begins tracking adjustments for the |
| 222 | /// uncounted path. |
| 223 | void beginElseRegion() { |
| 224 | RegionCount = ParentCount - Count; |
| 225 | PGO->setCurrentRegionCount(RegionCount); |
| 226 | } |
| 227 | |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 228 | /// Reset the current region count. |
| 229 | void setCurrentRegionCount(uint64_t CurrentCount) { |
| 230 | RegionCount = CurrentCount; |
| 231 | PGO->setCurrentRegionCount(RegionCount); |
| 232 | } |
| 233 | |
Justin Bogner | 0718a3a | 2014-01-13 21:24:22 +0000 | [diff] [blame] | 234 | /// Adjust for non-local control flow after emitting a subexpression or |
| 235 | /// substatement. This must be called to account for constructs such as gotos, |
| 236 | /// labels, and returns, so that we can ensure that our region's count is |
| 237 | /// correct in the code that follows. |
| 238 | void adjustForControlFlow() { |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 239 | Adjust += PGO->getCurrentRegionCount() - RegionCount; |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 240 | // Reset the region count in case this is called again later. |
| 241 | RegionCount = PGO->getCurrentRegionCount(); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 242 | } |
Bob Wilson | bf854f0 | 2014-02-17 19:21:09 +0000 | [diff] [blame] | 243 | |
| 244 | /// Commit all adjustments to the current region. If the region is a loop, |
| 245 | /// the LoopAdjust value should be the count of all the breaks and continues |
| 246 | /// from the loop, to compensate for those counts being deducted from the |
| 247 | /// adjustments for the body of the loop. |
| 248 | void applyAdjustmentsToRegion(uint64_t LoopAdjust) { |
| 249 | PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust); |
Justin Bogner | ef512b9 | 2014-01-06 22:27:43 +0000 | [diff] [blame] | 250 | } |
| 251 | }; |
| 252 | |
| 253 | } // end namespace CodeGen |
| 254 | } // end namespace clang |
| 255 | |
| 256 | #endif |