Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 1 | //===--- CodeGenPGO.cpp - PGO Instrumentation for LLVM CodeGen --*- C++ -*-===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // Instrumentation-based profile-guided optimization |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "CodeGenPGO.h" |
| 15 | #include "CodeGenFunction.h" |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 16 | #include "CoverageMappingGen.h" |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 17 | #include "clang/AST/RecursiveASTVisitor.h" |
| 18 | #include "clang/AST/StmtVisitor.h" |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 19 | #include "llvm/IR/Intrinsics.h" |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 20 | #include "llvm/IR/MDBuilder.h" |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 21 | #include "llvm/Support/Endian.h" |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 22 | #include "llvm/Support/FileSystem.h" |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 23 | #include "llvm/Support/MD5.h" |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 24 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 25 | static llvm::cl::opt<bool> EnableValueProfiling( |
| 26 | "enable-value-profiling", llvm::cl::ZeroOrMore, |
| 27 | llvm::cl::desc("Enable value profiling"), llvm::cl::init(false)); |
| 28 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 29 | using namespace clang; |
| 30 | using namespace CodeGen; |
| 31 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 32 | void CodeGenPGO::setFuncName(StringRef Name, |
| 33 | llvm::GlobalValue::LinkageTypes Linkage) { |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 34 | llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); |
| 35 | FuncName = llvm::getPGOFuncName( |
| 36 | Name, Linkage, CGM.getCodeGenOpts().MainFileName, |
| 37 | PGOReader ? PGOReader->getVersion() : llvm::IndexedInstrProf::Version); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 38 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 39 | // If we're generating a profile, create a variable for the name. |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 40 | if (CGM.getCodeGenOpts().hasProfileClangInstr()) |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 41 | FuncNameVar = llvm::createPGOFuncNameVar(CGM.getModule(), Linkage, FuncName); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 42 | } |
| 43 | |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 44 | void CodeGenPGO::setFuncName(llvm::Function *Fn) { |
| 45 | setFuncName(Fn->getName(), Fn->getLinkage()); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 46 | // Create PGOFuncName meta data. |
| 47 | llvm::createPGOFuncNameMetadata(*Fn, FuncName); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 48 | } |
| 49 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 50 | namespace { |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 51 | /// \brief Stable hasher for PGO region counters. |
| 52 | /// |
| 53 | /// PGOHash produces a stable hash of a given function's control flow. |
| 54 | /// |
| 55 | /// Changing the output of this hash will invalidate all previously generated |
| 56 | /// profiles -- i.e., don't do it. |
| 57 | /// |
| 58 | /// \note When this hash does eventually change (years?), we still need to |
| 59 | /// support old hashes. We'll need to pull in the version number from the |
| 60 | /// profile data format and use the matching hash function. |
| 61 | class PGOHash { |
| 62 | uint64_t Working; |
| 63 | unsigned Count; |
| 64 | llvm::MD5 MD5; |
| 65 | |
| 66 | static const int NumBitsPerType = 6; |
| 67 | static const unsigned NumTypesPerWord = sizeof(uint64_t) * 8 / NumBitsPerType; |
| 68 | static const unsigned TooBig = 1u << NumBitsPerType; |
| 69 | |
| 70 | public: |
| 71 | /// \brief Hash values for AST nodes. |
| 72 | /// |
| 73 | /// Distinct values for AST nodes that have region counters attached. |
| 74 | /// |
| 75 | /// These values must be stable. All new members must be added at the end, |
| 76 | /// and no members should be removed. Changing the enumeration value for an |
| 77 | /// AST node will affect the hash of every function that contains that node. |
| 78 | enum HashType : unsigned char { |
| 79 | None = 0, |
| 80 | LabelStmt = 1, |
| 81 | WhileStmt, |
| 82 | DoStmt, |
| 83 | ForStmt, |
| 84 | CXXForRangeStmt, |
| 85 | ObjCForCollectionStmt, |
| 86 | SwitchStmt, |
| 87 | CaseStmt, |
| 88 | DefaultStmt, |
| 89 | IfStmt, |
| 90 | CXXTryStmt, |
| 91 | CXXCatchStmt, |
| 92 | ConditionalOperator, |
| 93 | BinaryOperatorLAnd, |
| 94 | BinaryOperatorLOr, |
| 95 | BinaryConditionalOperator, |
| 96 | |
| 97 | // Keep this last. It's for the static assert that follows. |
| 98 | LastHashType |
| 99 | }; |
| 100 | static_assert(LastHashType <= TooBig, "Too many types in HashType"); |
| 101 | |
| 102 | // TODO: When this format changes, take in a version number here, and use the |
| 103 | // old hash calculation for file formats that used the old hash. |
| 104 | PGOHash() : Working(0), Count(0) {} |
| 105 | void combine(HashType Type); |
| 106 | uint64_t finalize(); |
| 107 | }; |
| 108 | const int PGOHash::NumBitsPerType; |
| 109 | const unsigned PGOHash::NumTypesPerWord; |
| 110 | const unsigned PGOHash::TooBig; |
| 111 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 112 | /// A RecursiveASTVisitor that fills a map of statements to PGO counters. |
| 113 | struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { |
| 114 | /// The next counter value to assign. |
| 115 | unsigned NextCounter; |
| 116 | /// The function hash. |
| 117 | PGOHash Hash; |
| 118 | /// The map of statements to counters. |
| 119 | llvm::DenseMap<const Stmt *, unsigned> &CounterMap; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 120 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 121 | MapRegionCounters(llvm::DenseMap<const Stmt *, unsigned> &CounterMap) |
| 122 | : NextCounter(0), CounterMap(CounterMap) {} |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 123 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 124 | // Blocks and lambdas are handled as separate functions, so we need not |
| 125 | // traverse them in the parent context. |
| 126 | bool TraverseBlockExpr(BlockExpr *BE) { return true; } |
| 127 | bool TraverseLambdaBody(LambdaExpr *LE) { return true; } |
| 128 | bool TraverseCapturedStmt(CapturedStmt *CS) { return true; } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 129 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 130 | bool VisitDecl(const Decl *D) { |
| 131 | switch (D->getKind()) { |
| 132 | default: |
| 133 | break; |
| 134 | case Decl::Function: |
| 135 | case Decl::CXXMethod: |
| 136 | case Decl::CXXConstructor: |
| 137 | case Decl::CXXDestructor: |
| 138 | case Decl::CXXConversion: |
| 139 | case Decl::ObjCMethod: |
| 140 | case Decl::Block: |
| 141 | case Decl::Captured: |
| 142 | CounterMap[D->getBody()] = NextCounter++; |
| 143 | break; |
| 144 | } |
| 145 | return true; |
| 146 | } |
| 147 | |
| 148 | bool VisitStmt(const Stmt *S) { |
| 149 | auto Type = getHashType(S); |
| 150 | if (Type == PGOHash::None) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 151 | return true; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 152 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 153 | CounterMap[S] = NextCounter++; |
| 154 | Hash.combine(Type); |
| 155 | return true; |
| 156 | } |
| 157 | PGOHash::HashType getHashType(const Stmt *S) { |
| 158 | switch (S->getStmtClass()) { |
| 159 | default: |
| 160 | break; |
| 161 | case Stmt::LabelStmtClass: |
| 162 | return PGOHash::LabelStmt; |
| 163 | case Stmt::WhileStmtClass: |
| 164 | return PGOHash::WhileStmt; |
| 165 | case Stmt::DoStmtClass: |
| 166 | return PGOHash::DoStmt; |
| 167 | case Stmt::ForStmtClass: |
| 168 | return PGOHash::ForStmt; |
| 169 | case Stmt::CXXForRangeStmtClass: |
| 170 | return PGOHash::CXXForRangeStmt; |
| 171 | case Stmt::ObjCForCollectionStmtClass: |
| 172 | return PGOHash::ObjCForCollectionStmt; |
| 173 | case Stmt::SwitchStmtClass: |
| 174 | return PGOHash::SwitchStmt; |
| 175 | case Stmt::CaseStmtClass: |
| 176 | return PGOHash::CaseStmt; |
| 177 | case Stmt::DefaultStmtClass: |
| 178 | return PGOHash::DefaultStmt; |
| 179 | case Stmt::IfStmtClass: |
| 180 | return PGOHash::IfStmt; |
| 181 | case Stmt::CXXTryStmtClass: |
| 182 | return PGOHash::CXXTryStmt; |
| 183 | case Stmt::CXXCatchStmtClass: |
| 184 | return PGOHash::CXXCatchStmt; |
| 185 | case Stmt::ConditionalOperatorClass: |
| 186 | return PGOHash::ConditionalOperator; |
| 187 | case Stmt::BinaryConditionalOperatorClass: |
| 188 | return PGOHash::BinaryConditionalOperator; |
| 189 | case Stmt::BinaryOperatorClass: { |
| 190 | const BinaryOperator *BO = cast<BinaryOperator>(S); |
| 191 | if (BO->getOpcode() == BO_LAnd) |
| 192 | return PGOHash::BinaryOperatorLAnd; |
| 193 | if (BO->getOpcode() == BO_LOr) |
| 194 | return PGOHash::BinaryOperatorLOr; |
| 195 | break; |
| 196 | } |
| 197 | } |
| 198 | return PGOHash::None; |
| 199 | } |
| 200 | }; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 201 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 202 | /// A StmtVisitor that propagates the raw counts through the AST and |
| 203 | /// records the count at statements where the value may change. |
| 204 | struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { |
| 205 | /// PGO state. |
| 206 | CodeGenPGO &PGO; |
| 207 | |
| 208 | /// A flag that is set when the current count should be recorded on the |
| 209 | /// next statement, such as at the exit of a loop. |
| 210 | bool RecordNextStmtCount; |
| 211 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 212 | /// The count at the current location in the traversal. |
| 213 | uint64_t CurrentCount; |
| 214 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 215 | /// The map of statements to count values. |
| 216 | llvm::DenseMap<const Stmt *, uint64_t> &CountMap; |
| 217 | |
| 218 | /// BreakContinueStack - Keep counts of breaks and continues inside loops. |
| 219 | struct BreakContinue { |
| 220 | uint64_t BreakCount; |
| 221 | uint64_t ContinueCount; |
| 222 | BreakContinue() : BreakCount(0), ContinueCount(0) {} |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 223 | }; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 224 | SmallVector<BreakContinue, 8> BreakContinueStack; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 225 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 226 | ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &CountMap, |
| 227 | CodeGenPGO &PGO) |
| 228 | : PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {} |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 229 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 230 | void RecordStmtCount(const Stmt *S) { |
| 231 | if (RecordNextStmtCount) { |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 232 | CountMap[S] = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 233 | RecordNextStmtCount = false; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 234 | } |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 235 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 236 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 237 | /// Set and return the current count. |
| 238 | uint64_t setCount(uint64_t Count) { |
| 239 | CurrentCount = Count; |
| 240 | return Count; |
| 241 | } |
| 242 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 243 | void VisitStmt(const Stmt *S) { |
| 244 | RecordStmtCount(S); |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 245 | for (const Stmt *Child : S->children()) |
| 246 | if (Child) |
| 247 | this->Visit(Child); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 248 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 249 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 250 | void VisitFunctionDecl(const FunctionDecl *D) { |
| 251 | // Counter tracks entry to the function body. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 252 | uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 253 | CountMap[D->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 254 | Visit(D->getBody()); |
| 255 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 256 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 257 | // Skip lambda expressions. We visit these as FunctionDecls when we're |
| 258 | // generating them and aren't interested in the body when generating a |
| 259 | // parent context. |
| 260 | void VisitLambdaExpr(const LambdaExpr *LE) {} |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 261 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 262 | void VisitCapturedDecl(const CapturedDecl *D) { |
| 263 | // Counter tracks entry to the capture body. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 264 | uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 265 | CountMap[D->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 266 | Visit(D->getBody()); |
| 267 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 268 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 269 | void VisitObjCMethodDecl(const ObjCMethodDecl *D) { |
| 270 | // Counter tracks entry to the method body. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 271 | uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 272 | CountMap[D->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 273 | Visit(D->getBody()); |
| 274 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 275 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 276 | void VisitBlockDecl(const BlockDecl *D) { |
| 277 | // Counter tracks entry to the block body. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 278 | uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 279 | CountMap[D->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 280 | Visit(D->getBody()); |
| 281 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 282 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 283 | void VisitReturnStmt(const ReturnStmt *S) { |
| 284 | RecordStmtCount(S); |
| 285 | if (S->getRetValue()) |
| 286 | Visit(S->getRetValue()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 287 | CurrentCount = 0; |
| 288 | RecordNextStmtCount = true; |
| 289 | } |
| 290 | |
| 291 | void VisitCXXThrowExpr(const CXXThrowExpr *E) { |
| 292 | RecordStmtCount(E); |
| 293 | if (E->getSubExpr()) |
| 294 | Visit(E->getSubExpr()); |
| 295 | CurrentCount = 0; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 296 | RecordNextStmtCount = true; |
| 297 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 298 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 299 | void VisitGotoStmt(const GotoStmt *S) { |
| 300 | RecordStmtCount(S); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 301 | CurrentCount = 0; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 302 | RecordNextStmtCount = true; |
| 303 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 304 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 305 | void VisitLabelStmt(const LabelStmt *S) { |
| 306 | RecordNextStmtCount = false; |
| 307 | // Counter tracks the block following the label. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 308 | uint64_t BlockCount = setCount(PGO.getRegionCount(S)); |
| 309 | CountMap[S] = BlockCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 310 | Visit(S->getSubStmt()); |
| 311 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 312 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 313 | void VisitBreakStmt(const BreakStmt *S) { |
| 314 | RecordStmtCount(S); |
| 315 | assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 316 | BreakContinueStack.back().BreakCount += CurrentCount; |
| 317 | CurrentCount = 0; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 318 | RecordNextStmtCount = true; |
| 319 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 320 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 321 | void VisitContinueStmt(const ContinueStmt *S) { |
| 322 | RecordStmtCount(S); |
| 323 | assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 324 | BreakContinueStack.back().ContinueCount += CurrentCount; |
| 325 | CurrentCount = 0; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 326 | RecordNextStmtCount = true; |
| 327 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 328 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 329 | void VisitWhileStmt(const WhileStmt *S) { |
| 330 | RecordStmtCount(S); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 331 | uint64_t ParentCount = CurrentCount; |
| 332 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 333 | BreakContinueStack.push_back(BreakContinue()); |
| 334 | // Visit the body region first so the break/continue adjustments can be |
| 335 | // included when visiting the condition. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 336 | uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 337 | CountMap[S->getBody()] = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 338 | Visit(S->getBody()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 339 | uint64_t BackedgeCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 340 | |
| 341 | // ...then go back and propagate counts through the condition. The count |
| 342 | // at the start of the condition is the sum of the incoming edges, |
| 343 | // the backedge from the end of the loop body, and the edges from |
| 344 | // continue statements. |
| 345 | BreakContinue BC = BreakContinueStack.pop_back_val(); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 346 | uint64_t CondCount = |
| 347 | setCount(ParentCount + BackedgeCount + BC.ContinueCount); |
| 348 | CountMap[S->getCond()] = CondCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 349 | Visit(S->getCond()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 350 | setCount(BC.BreakCount + CondCount - BodyCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 351 | RecordNextStmtCount = true; |
| 352 | } |
| 353 | |
| 354 | void VisitDoStmt(const DoStmt *S) { |
| 355 | RecordStmtCount(S); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 356 | uint64_t LoopCount = PGO.getRegionCount(S); |
| 357 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 358 | BreakContinueStack.push_back(BreakContinue()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 359 | // The count doesn't include the fallthrough from the parent scope. Add it. |
| 360 | uint64_t BodyCount = setCount(LoopCount + CurrentCount); |
| 361 | CountMap[S->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 362 | Visit(S->getBody()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 363 | uint64_t BackedgeCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 364 | |
| 365 | BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 366 | // The count at the start of the condition is equal to the count at the |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 367 | // end of the body, plus any continues. |
| 368 | uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount); |
| 369 | CountMap[S->getCond()] = CondCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 370 | Visit(S->getCond()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 371 | setCount(BC.BreakCount + CondCount - LoopCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 372 | RecordNextStmtCount = true; |
| 373 | } |
| 374 | |
| 375 | void VisitForStmt(const ForStmt *S) { |
| 376 | RecordStmtCount(S); |
| 377 | if (S->getInit()) |
| 378 | Visit(S->getInit()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 379 | |
| 380 | uint64_t ParentCount = CurrentCount; |
| 381 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 382 | BreakContinueStack.push_back(BreakContinue()); |
| 383 | // Visit the body region first. (This is basically the same as a while |
| 384 | // loop; see further comments in VisitWhileStmt.) |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 385 | uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 386 | CountMap[S->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 387 | Visit(S->getBody()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 388 | uint64_t BackedgeCount = CurrentCount; |
| 389 | BreakContinue BC = BreakContinueStack.pop_back_val(); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 390 | |
| 391 | // The increment is essentially part of the body but it needs to include |
| 392 | // the count for all the continue statements. |
| 393 | if (S->getInc()) { |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 394 | uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); |
| 395 | CountMap[S->getInc()] = IncCount; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 396 | Visit(S->getInc()); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 397 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 398 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 399 | // ...then go back and propagate counts through the condition. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 400 | uint64_t CondCount = |
| 401 | setCount(ParentCount + BackedgeCount + BC.ContinueCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 402 | if (S->getCond()) { |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 403 | CountMap[S->getCond()] = CondCount; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 404 | Visit(S->getCond()); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 405 | } |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 406 | setCount(BC.BreakCount + CondCount - BodyCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 407 | RecordNextStmtCount = true; |
| 408 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 409 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 410 | void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { |
| 411 | RecordStmtCount(S); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 412 | Visit(S->getLoopVarStmt()); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 413 | Visit(S->getRangeStmt()); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 414 | Visit(S->getBeginStmt()); |
| 415 | Visit(S->getEndStmt()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 416 | |
| 417 | uint64_t ParentCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 418 | BreakContinueStack.push_back(BreakContinue()); |
| 419 | // Visit the body region first. (This is basically the same as a while |
| 420 | // loop; see further comments in VisitWhileStmt.) |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 421 | uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 422 | CountMap[S->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 423 | Visit(S->getBody()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 424 | uint64_t BackedgeCount = CurrentCount; |
| 425 | BreakContinue BC = BreakContinueStack.pop_back_val(); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 426 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 427 | // The increment is essentially part of the body but it needs to include |
| 428 | // the count for all the continue statements. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 429 | uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); |
| 430 | CountMap[S->getInc()] = IncCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 431 | Visit(S->getInc()); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 432 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 433 | // ...then go back and propagate counts through the condition. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 434 | uint64_t CondCount = |
| 435 | setCount(ParentCount + BackedgeCount + BC.ContinueCount); |
| 436 | CountMap[S->getCond()] = CondCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 437 | Visit(S->getCond()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 438 | setCount(BC.BreakCount + CondCount - BodyCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 439 | RecordNextStmtCount = true; |
| 440 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 441 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 442 | void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { |
| 443 | RecordStmtCount(S); |
| 444 | Visit(S->getElement()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 445 | uint64_t ParentCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 446 | BreakContinueStack.push_back(BreakContinue()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 447 | // Counter tracks the body of the loop. |
| 448 | uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 449 | CountMap[S->getBody()] = BodyCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 450 | Visit(S->getBody()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 451 | uint64_t BackedgeCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 452 | BreakContinue BC = BreakContinueStack.pop_back_val(); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 453 | |
| 454 | setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount - |
| 455 | BodyCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 456 | RecordNextStmtCount = true; |
| 457 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 458 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 459 | void VisitSwitchStmt(const SwitchStmt *S) { |
| 460 | RecordStmtCount(S); |
| 461 | Visit(S->getCond()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 462 | CurrentCount = 0; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 463 | BreakContinueStack.push_back(BreakContinue()); |
| 464 | Visit(S->getBody()); |
| 465 | // If the switch is inside a loop, add the continue counts. |
| 466 | BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 467 | if (!BreakContinueStack.empty()) |
| 468 | BreakContinueStack.back().ContinueCount += BC.ContinueCount; |
| 469 | // Counter tracks the exit block of the switch. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 470 | setCount(PGO.getRegionCount(S)); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 471 | RecordNextStmtCount = true; |
| 472 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 473 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 474 | void VisitSwitchCase(const SwitchCase *S) { |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 475 | RecordNextStmtCount = false; |
| 476 | // Counter for this particular case. This counts only jumps from the |
| 477 | // switch header and does not include fallthrough from the case before |
| 478 | // this one. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 479 | uint64_t CaseCount = PGO.getRegionCount(S); |
| 480 | setCount(CurrentCount + CaseCount); |
| 481 | // We need the count without fallthrough in the mapping, so it's more useful |
| 482 | // for branch probabilities. |
| 483 | CountMap[S] = CaseCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 484 | RecordNextStmtCount = true; |
| 485 | Visit(S->getSubStmt()); |
| 486 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 487 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 488 | void VisitIfStmt(const IfStmt *S) { |
| 489 | RecordStmtCount(S); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 490 | uint64_t ParentCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 491 | Visit(S->getCond()); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 492 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 493 | // Counter tracks the "then" part of an if statement. The count for |
| 494 | // the "else" part, if it exists, will be calculated from this counter. |
| 495 | uint64_t ThenCount = setCount(PGO.getRegionCount(S)); |
| 496 | CountMap[S->getThen()] = ThenCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 497 | Visit(S->getThen()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 498 | uint64_t OutCount = CurrentCount; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 499 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 500 | uint64_t ElseCount = ParentCount - ThenCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 501 | if (S->getElse()) { |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 502 | setCount(ElseCount); |
| 503 | CountMap[S->getElse()] = ElseCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 504 | Visit(S->getElse()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 505 | OutCount += CurrentCount; |
| 506 | } else |
| 507 | OutCount += ElseCount; |
| 508 | setCount(OutCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 509 | RecordNextStmtCount = true; |
| 510 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 511 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 512 | void VisitCXXTryStmt(const CXXTryStmt *S) { |
| 513 | RecordStmtCount(S); |
| 514 | Visit(S->getTryBlock()); |
| 515 | for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) |
| 516 | Visit(S->getHandler(I)); |
| 517 | // Counter tracks the continuation block of the try statement. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 518 | setCount(PGO.getRegionCount(S)); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 519 | RecordNextStmtCount = true; |
| 520 | } |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 521 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 522 | void VisitCXXCatchStmt(const CXXCatchStmt *S) { |
| 523 | RecordNextStmtCount = false; |
| 524 | // Counter tracks the catch statement's handler block. |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 525 | uint64_t CatchCount = setCount(PGO.getRegionCount(S)); |
| 526 | CountMap[S] = CatchCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 527 | Visit(S->getHandlerBlock()); |
| 528 | } |
| 529 | |
| 530 | void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { |
| 531 | RecordStmtCount(E); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 532 | uint64_t ParentCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 533 | Visit(E->getCond()); |
| 534 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 535 | // Counter tracks the "true" part of a conditional operator. The |
| 536 | // count in the "false" part will be calculated from this counter. |
| 537 | uint64_t TrueCount = setCount(PGO.getRegionCount(E)); |
| 538 | CountMap[E->getTrueExpr()] = TrueCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 539 | Visit(E->getTrueExpr()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 540 | uint64_t OutCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 541 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 542 | uint64_t FalseCount = setCount(ParentCount - TrueCount); |
| 543 | CountMap[E->getFalseExpr()] = FalseCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 544 | Visit(E->getFalseExpr()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 545 | OutCount += CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 546 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 547 | setCount(OutCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 548 | RecordNextStmtCount = true; |
| 549 | } |
| 550 | |
| 551 | void VisitBinLAnd(const BinaryOperator *E) { |
| 552 | RecordStmtCount(E); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 553 | uint64_t ParentCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 554 | Visit(E->getLHS()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 555 | // Counter tracks the right hand side of a logical and operator. |
| 556 | uint64_t RHSCount = setCount(PGO.getRegionCount(E)); |
| 557 | CountMap[E->getRHS()] = RHSCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 558 | Visit(E->getRHS()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 559 | setCount(ParentCount + RHSCount - CurrentCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 560 | RecordNextStmtCount = true; |
| 561 | } |
| 562 | |
| 563 | void VisitBinLOr(const BinaryOperator *E) { |
| 564 | RecordStmtCount(E); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 565 | uint64_t ParentCount = CurrentCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 566 | Visit(E->getLHS()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 567 | // Counter tracks the right hand side of a logical or operator. |
| 568 | uint64_t RHSCount = setCount(PGO.getRegionCount(E)); |
| 569 | CountMap[E->getRHS()] = RHSCount; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 570 | Visit(E->getRHS()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 571 | setCount(ParentCount + RHSCount - CurrentCount); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 572 | RecordNextStmtCount = true; |
| 573 | } |
| 574 | }; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 575 | } // end anonymous namespace |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 576 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 577 | void PGOHash::combine(HashType Type) { |
| 578 | // Check that we never combine 0 and only have six bits. |
| 579 | assert(Type && "Hash is invalid: unexpected type 0"); |
| 580 | assert(unsigned(Type) < TooBig && "Hash is invalid: too many types"); |
| 581 | |
| 582 | // Pass through MD5 if enough work has built up. |
| 583 | if (Count && Count % NumTypesPerWord == 0) { |
| 584 | using namespace llvm::support; |
| 585 | uint64_t Swapped = endian::byte_swap<uint64_t, little>(Working); |
| 586 | MD5.update(llvm::makeArrayRef((uint8_t *)&Swapped, sizeof(Swapped))); |
| 587 | Working = 0; |
| 588 | } |
| 589 | |
| 590 | // Accumulate the current type. |
| 591 | ++Count; |
| 592 | Working = Working << NumBitsPerType | Type; |
| 593 | } |
| 594 | |
| 595 | uint64_t PGOHash::finalize() { |
| 596 | // Use Working as the hash directly if we never used MD5. |
| 597 | if (Count <= NumTypesPerWord) |
| 598 | // No need to byte swap here, since none of the math was endian-dependent. |
| 599 | // This number will be byte-swapped as required on endianness transitions, |
| 600 | // so we will see the same value on the other side. |
| 601 | return Working; |
| 602 | |
| 603 | // Check for remaining work in Working. |
| 604 | if (Working) |
| 605 | MD5.update(Working); |
| 606 | |
| 607 | // Finalize the MD5 and return the hash. |
| 608 | llvm::MD5::MD5Result Result; |
| 609 | MD5.final(Result); |
| 610 | using namespace llvm::support; |
| 611 | return endian::read<uint64_t, little, unaligned>(Result); |
| 612 | } |
| 613 | |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 614 | void CodeGenPGO::assignRegionCounters(GlobalDecl GD, llvm::Function *Fn) { |
| 615 | const Decl *D = GD.getDecl(); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 616 | bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr(); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 617 | llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); |
| 618 | if (!InstrumentRegions && !PGOReader) |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 619 | return; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 620 | if (D->isImplicit()) |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 621 | return; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 622 | // Constructors and destructors may be represented by several functions in IR. |
| 623 | // If so, instrument only base variant, others are implemented by delegation |
| 624 | // to the base one, it would be counted twice otherwise. |
| 625 | if (CGM.getTarget().getCXXABI().hasConstructorVariants() && |
| 626 | ((isa<CXXConstructorDecl>(GD.getDecl()) && |
| 627 | GD.getCtorType() != Ctor_Base) || |
| 628 | (isa<CXXDestructorDecl>(GD.getDecl()) && |
| 629 | GD.getDtorType() != Dtor_Base))) { |
| 630 | return; |
| 631 | } |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 632 | CGM.ClearUnusedCoverageMapping(D); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 633 | setFuncName(Fn); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 634 | |
| 635 | mapRegionCounters(D); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 636 | if (CGM.getCodeGenOpts().CoverageMapping) |
| 637 | emitCounterRegionMapping(D); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 638 | if (PGOReader) { |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 639 | SourceManager &SM = CGM.getContext().getSourceManager(); |
| 640 | loadRegionCounts(PGOReader, SM.isInMainFile(D->getLocation())); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 641 | computeRegionCounts(D); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 642 | applyFunctionAttributes(PGOReader, Fn); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 643 | } |
| 644 | } |
| 645 | |
| 646 | void CodeGenPGO::mapRegionCounters(const Decl *D) { |
| 647 | RegionCounterMap.reset(new llvm::DenseMap<const Stmt *, unsigned>); |
| 648 | MapRegionCounters Walker(*RegionCounterMap); |
| 649 | if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 650 | Walker.TraverseDecl(const_cast<FunctionDecl *>(FD)); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 651 | else if (const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D)) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 652 | Walker.TraverseDecl(const_cast<ObjCMethodDecl *>(MD)); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 653 | else if (const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D)) |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 654 | Walker.TraverseDecl(const_cast<BlockDecl *>(BD)); |
| 655 | else if (const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D)) |
| 656 | Walker.TraverseDecl(const_cast<CapturedDecl *>(CD)); |
| 657 | assert(Walker.NextCounter > 0 && "no entry counter mapped for decl"); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 658 | NumRegionCounters = Walker.NextCounter; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 659 | FunctionHash = Walker.Hash.finalize(); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 660 | } |
| 661 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 662 | bool CodeGenPGO::skipRegionMappingForDecl(const Decl *D) { |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 663 | if (SkipCoverageMapping) |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 664 | return true; |
| 665 | |
| 666 | // Don't map the functions in system headers. |
| 667 | const auto &SM = CGM.getContext().getSourceManager(); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 668 | auto Loc = D->getBody()->getLocStart(); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 669 | return SM.isInSystemHeader(Loc); |
| 670 | } |
| 671 | |
| 672 | void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { |
| 673 | if (skipRegionMappingForDecl(D)) |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 674 | return; |
| 675 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 676 | std::string CoverageMapping; |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 677 | llvm::raw_string_ostream OS(CoverageMapping); |
| 678 | CoverageMappingGen MappingGen(*CGM.getCoverageMapping(), |
| 679 | CGM.getContext().getSourceManager(), |
| 680 | CGM.getLangOpts(), RegionCounterMap.get()); |
| 681 | MappingGen.emitCounterMapping(D, OS); |
| 682 | OS.flush(); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 683 | |
| 684 | if (CoverageMapping.empty()) |
| 685 | return; |
| 686 | |
| 687 | CGM.getCoverageMapping()->addFunctionMappingRecord( |
| 688 | FuncNameVar, FuncName, FunctionHash, CoverageMapping); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 689 | } |
| 690 | |
| 691 | void |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 692 | CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef Name, |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 693 | llvm::GlobalValue::LinkageTypes Linkage) { |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 694 | if (skipRegionMappingForDecl(D)) |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 695 | return; |
| 696 | |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 697 | std::string CoverageMapping; |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 698 | llvm::raw_string_ostream OS(CoverageMapping); |
| 699 | CoverageMappingGen MappingGen(*CGM.getCoverageMapping(), |
| 700 | CGM.getContext().getSourceManager(), |
| 701 | CGM.getLangOpts()); |
| 702 | MappingGen.emitEmptyMapping(D, OS); |
| 703 | OS.flush(); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 704 | |
| 705 | if (CoverageMapping.empty()) |
| 706 | return; |
| 707 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 708 | setFuncName(Name, Linkage); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 709 | CGM.getCoverageMapping()->addFunctionMappingRecord( |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 710 | FuncNameVar, FuncName, FunctionHash, CoverageMapping, false); |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 711 | } |
| 712 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 713 | void CodeGenPGO::computeRegionCounts(const Decl *D) { |
| 714 | StmtCountMap.reset(new llvm::DenseMap<const Stmt *, uint64_t>); |
| 715 | ComputeRegionCounts Walker(*StmtCountMap, *this); |
| 716 | if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) |
| 717 | Walker.VisitFunctionDecl(FD); |
| 718 | else if (const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D)) |
| 719 | Walker.VisitObjCMethodDecl(MD); |
| 720 | else if (const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D)) |
| 721 | Walker.VisitBlockDecl(BD); |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 722 | else if (const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D)) |
| 723 | Walker.VisitCapturedDecl(const_cast<CapturedDecl *>(CD)); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 724 | } |
| 725 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 726 | void |
| 727 | CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, |
| 728 | llvm::Function *Fn) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 729 | if (!haveRegionCounts()) |
| 730 | return; |
| 731 | |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 732 | uint64_t FunctionCount = getRegionCount(nullptr); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 733 | Fn->setEntryCount(FunctionCount); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 734 | } |
| 735 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 736 | void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) { |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 737 | if (!CGM.getCodeGenOpts().hasProfileClangInstr() || !RegionCounterMap) |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 738 | return; |
Pirama Arumuga Nainar | 87d948e | 2016-03-03 15:49:35 -0800 | [diff] [blame] | 739 | if (!Builder.GetInsertBlock()) |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 740 | return; |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 741 | |
| 742 | unsigned Counter = (*RegionCounterMap)[S]; |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 743 | auto *I8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext()); |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 744 | Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), |
| 745 | {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 746 | Builder.getInt64(FunctionHash), |
| 747 | Builder.getInt32(NumRegionCounters), |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 748 | Builder.getInt32(Counter)}); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 749 | } |
| 750 | |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 751 | // This method either inserts a call to the profile run-time during |
| 752 | // instrumentation or puts profile data into metadata for PGO use. |
| 753 | void CodeGenPGO::valueProfile(CGBuilderTy &Builder, uint32_t ValueKind, |
| 754 | llvm::Instruction *ValueSite, llvm::Value *ValuePtr) { |
| 755 | |
| 756 | if (!EnableValueProfiling) |
| 757 | return; |
| 758 | |
| 759 | if (!ValuePtr || !ValueSite || !Builder.GetInsertBlock()) |
| 760 | return; |
| 761 | |
| 762 | if (isa<llvm::Constant>(ValuePtr)) |
| 763 | return; |
| 764 | |
| 765 | bool InstrumentValueSites = CGM.getCodeGenOpts().hasProfileClangInstr(); |
| 766 | if (InstrumentValueSites && RegionCounterMap) { |
| 767 | auto BuilderInsertPoint = Builder.saveIP(); |
| 768 | Builder.SetInsertPoint(ValueSite); |
| 769 | llvm::Value *Args[5] = { |
| 770 | llvm::ConstantExpr::getBitCast(FuncNameVar, Builder.getInt8PtrTy()), |
| 771 | Builder.getInt64(FunctionHash), |
| 772 | Builder.CreatePtrToInt(ValuePtr, Builder.getInt64Ty()), |
| 773 | Builder.getInt32(ValueKind), |
| 774 | Builder.getInt32(NumValueSites[ValueKind]++) |
| 775 | }; |
| 776 | Builder.CreateCall( |
| 777 | CGM.getIntrinsic(llvm::Intrinsic::instrprof_value_profile), Args); |
| 778 | Builder.restoreIP(BuilderInsertPoint); |
| 779 | return; |
| 780 | } |
| 781 | |
| 782 | llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); |
| 783 | if (PGOReader && haveRegionCounts()) { |
| 784 | // We record the top most called three functions at each call site. |
| 785 | // Profile metadata contains "VP" string identifying this metadata |
| 786 | // as value profiling data, then a uint32_t value for the value profiling |
| 787 | // kind, a uint64_t value for the total number of times the call is |
| 788 | // executed, followed by the function hash and execution count (uint64_t) |
| 789 | // pairs for each function. |
| 790 | if (NumValueSites[ValueKind] >= ProfRecord->getNumValueSites(ValueKind)) |
| 791 | return; |
| 792 | |
| 793 | llvm::annotateValueSite(CGM.getModule(), *ValueSite, *ProfRecord, |
| 794 | (llvm::InstrProfValueKind)ValueKind, |
| 795 | NumValueSites[ValueKind]); |
| 796 | |
| 797 | NumValueSites[ValueKind]++; |
| 798 | } |
| 799 | } |
| 800 | |
Stephen Hines | c568f1e | 2014-07-21 00:47:37 -0700 | [diff] [blame] | 801 | void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, |
| 802 | bool IsInMainFile) { |
| 803 | CGM.getPGOStats().addVisited(IsInMainFile); |
Stephen Hines | 0e2c34f | 2015-03-23 12:09:02 -0700 | [diff] [blame] | 804 | RegionCounts.clear(); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 805 | llvm::Expected<llvm::InstrProfRecord> RecordExpected = |
| 806 | PGOReader->getInstrProfRecord(FuncName, FunctionHash); |
| 807 | if (auto E = RecordExpected.takeError()) { |
| 808 | auto IPE = llvm::InstrProfError::take(std::move(E)); |
| 809 | if (IPE == llvm::instrprof_error::unknown_function) |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 810 | CGM.getPGOStats().addMissing(IsInMainFile); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 811 | else if (IPE == llvm::instrprof_error::hash_mismatch) |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 812 | CGM.getPGOStats().addMismatched(IsInMainFile); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 813 | else if (IPE == llvm::instrprof_error::malformed) |
Stephen Hines | 176edba | 2014-12-01 14:53:08 -0800 | [diff] [blame] | 814 | // TODO: Consider a more specific warning for this case. |
| 815 | CGM.getPGOStats().addMismatched(IsInMainFile); |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 816 | return; |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 817 | } |
Pirama Arumuga Nainar | 4967a71 | 2016-09-19 22:19:55 -0700 | [diff] [blame^] | 818 | ProfRecord = |
| 819 | llvm::make_unique<llvm::InstrProfRecord>(std::move(RecordExpected.get())); |
| 820 | RegionCounts = ProfRecord->Counts; |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 821 | } |
| 822 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 823 | /// \brief Calculate what to divide by to scale weights. |
| 824 | /// |
| 825 | /// Given the maximum weight, calculate a divisor that will scale all the |
| 826 | /// weights to strictly less than UINT32_MAX. |
| 827 | static uint64_t calculateWeightScale(uint64_t MaxWeight) { |
| 828 | return MaxWeight < UINT32_MAX ? 1 : MaxWeight / UINT32_MAX + 1; |
| 829 | } |
| 830 | |
| 831 | /// \brief Scale an individual branch weight (and add 1). |
| 832 | /// |
| 833 | /// Scale a 64-bit weight down to 32-bits using \c Scale. |
| 834 | /// |
| 835 | /// According to Laplace's Rule of Succession, it is better to compute the |
| 836 | /// weight based on the count plus 1, so universally add 1 to the value. |
| 837 | /// |
| 838 | /// \pre \c Scale was calculated by \a calculateWeightScale() with a weight no |
| 839 | /// greater than \c Weight. |
| 840 | static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale) { |
| 841 | assert(Scale && "scale by 0?"); |
| 842 | uint64_t Scaled = Weight / Scale + 1; |
| 843 | assert(Scaled <= UINT32_MAX && "overflow 32-bits"); |
| 844 | return Scaled; |
| 845 | } |
| 846 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 847 | llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount, |
| 848 | uint64_t FalseCount) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 849 | // Check for empty weights. |
| 850 | if (!TrueCount && !FalseCount) |
| 851 | return nullptr; |
| 852 | |
| 853 | // Calculate how to scale down to 32-bits. |
| 854 | uint64_t Scale = calculateWeightScale(std::max(TrueCount, FalseCount)); |
| 855 | |
| 856 | llvm::MDBuilder MDHelper(CGM.getLLVMContext()); |
| 857 | return MDHelper.createBranchWeights(scaleBranchWeight(TrueCount, Scale), |
| 858 | scaleBranchWeight(FalseCount, Scale)); |
| 859 | } |
| 860 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 861 | llvm::MDNode * |
| 862 | CodeGenFunction::createProfileWeights(ArrayRef<uint64_t> Weights) { |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 863 | // We need at least two elements to create meaningful weights. |
| 864 | if (Weights.size() < 2) |
| 865 | return nullptr; |
| 866 | |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 867 | // Check for empty weights. |
| 868 | uint64_t MaxWeight = *std::max_element(Weights.begin(), Weights.end()); |
| 869 | if (MaxWeight == 0) |
| 870 | return nullptr; |
| 871 | |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 872 | // Calculate how to scale down to 32-bits. |
Stephen Hines | 6bcf27b | 2014-05-29 04:14:42 -0700 | [diff] [blame] | 873 | uint64_t Scale = calculateWeightScale(MaxWeight); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 874 | |
| 875 | SmallVector<uint32_t, 16> ScaledWeights; |
| 876 | ScaledWeights.reserve(Weights.size()); |
| 877 | for (uint64_t W : Weights) |
| 878 | ScaledWeights.push_back(scaleBranchWeight(W, Scale)); |
| 879 | |
| 880 | llvm::MDBuilder MDHelper(CGM.getLLVMContext()); |
| 881 | return MDHelper.createBranchWeights(ScaledWeights); |
| 882 | } |
| 883 | |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 884 | llvm::MDNode *CodeGenFunction::createProfileWeightsForLoop(const Stmt *Cond, |
| 885 | uint64_t LoopCount) { |
| 886 | if (!PGO.haveRegionCounts()) |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 887 | return nullptr; |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 888 | Optional<uint64_t> CondCount = PGO.getStmtCount(Cond); |
Pirama Arumuga Nainar | 58878f8 | 2015-05-06 11:48:57 -0700 | [diff] [blame] | 889 | assert(CondCount.hasValue() && "missing expected loop condition count"); |
| 890 | if (*CondCount == 0) |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 891 | return nullptr; |
Pirama Arumuga Nainar | b6d6993 | 2015-07-01 12:25:36 -0700 | [diff] [blame] | 892 | return createProfileWeights(LoopCount, |
| 893 | std::max(*CondCount, LoopCount) - LoopCount); |
Stephen Hines | 651f13c | 2014-04-23 16:59:28 -0700 | [diff] [blame] | 894 | } |