Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 1 | //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 10 | // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling. |
| 11 | // It also builds the data structures and initialization code needed for |
| 12 | // updating execution counts and emitting the profile at runtime. |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 13 | // |
| 14 | //===----------------------------------------------------------------------===// |
| 15 | |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 16 | #include "llvm/Transforms/InstrProfiling.h" |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 17 | #include "llvm/ADT/Triple.h" |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 18 | #include "llvm/Analysis/TargetLibraryInfo.h" |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 19 | #include "llvm/IR/IRBuilder.h" |
| 20 | #include "llvm/IR/IntrinsicInst.h" |
| 21 | #include "llvm/IR/Module.h" |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 22 | #include "llvm/ProfileData/InstrProf.h" |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 23 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
| 24 | |
| 25 | using namespace llvm; |
| 26 | |
| 27 | #define DEBUG_TYPE "instrprof" |
| 28 | |
| 29 | namespace { |
| 30 | |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 31 | cl::opt<bool> DoNameCompression("enable-name-compression", |
| 32 | cl::desc("Enable name string compression"), |
| 33 | cl::init(true)); |
| 34 | |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 35 | cl::opt<bool> ValueProfileStaticAlloc( |
| 36 | "vp-static-alloc", |
| 37 | cl::desc("Do static counter allocation for value profiler"), |
| 38 | cl::init(true)); |
| 39 | cl::opt<double> NumCountersPerValueSite( |
| 40 | "vp-counters-per-site", |
| 41 | cl::desc("The average number of profile counters allocated " |
| 42 | "per value profiling site."), |
| 43 | // This is set to a very small value because in real programs, only |
| 44 | // a very small percentage of value sites have non-zero targets, e.g, 1/30. |
| 45 | // For those sites with non-zero profile, the average number of targets |
| 46 | // is usually smaller than 2. |
| 47 | cl::init(1.0)); |
| 48 | |
Xinliang David Li | e6b8929 | 2016-04-18 17:47:38 +0000 | [diff] [blame] | 49 | class InstrProfilingLegacyPass : public ModulePass { |
| 50 | InstrProfiling InstrProf; |
| 51 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 52 | public: |
| 53 | static char ID; |
Xinliang David Li | e6b8929 | 2016-04-18 17:47:38 +0000 | [diff] [blame] | 54 | InstrProfilingLegacyPass() : ModulePass(ID), InstrProf() {} |
| 55 | InstrProfilingLegacyPass(const InstrProfOptions &Options) |
| 56 | : ModulePass(ID), InstrProf(Options) {} |
Mehdi Amini | 117296c | 2016-10-01 02:56:57 +0000 | [diff] [blame] | 57 | StringRef getPassName() const override { |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 58 | return "Frontend instrumentation-based coverage lowering"; |
| 59 | } |
| 60 | |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 61 | bool runOnModule(Module &M) override { |
| 62 | return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI()); |
| 63 | } |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 64 | |
| 65 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 66 | AU.setPreservesCFG(); |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 67 | AU.addRequired<TargetLibraryInfoWrapperPass>(); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 68 | } |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 69 | }; |
| 70 | |
| 71 | } // anonymous namespace |
| 72 | |
Sean Silva | fd03ac6 | 2016-08-09 00:28:38 +0000 | [diff] [blame] | 73 | PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) { |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 74 | auto &TLI = AM.getResult<TargetLibraryAnalysis>(M); |
| 75 | if (!run(M, TLI)) |
Xinliang David Li | e6b8929 | 2016-04-18 17:47:38 +0000 | [diff] [blame] | 76 | return PreservedAnalyses::all(); |
| 77 | |
| 78 | return PreservedAnalyses::none(); |
| 79 | } |
| 80 | |
| 81 | char InstrProfilingLegacyPass::ID = 0; |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 82 | INITIALIZE_PASS_BEGIN( |
| 83 | InstrProfilingLegacyPass, "instrprof", |
| 84 | "Frontend instrumentation-based coverage lowering.", false, false) |
| 85 | INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) |
| 86 | INITIALIZE_PASS_END( |
| 87 | InstrProfilingLegacyPass, "instrprof", |
| 88 | "Frontend instrumentation-based coverage lowering.", false, false) |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 89 | |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 90 | ModulePass * |
| 91 | llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) { |
Xinliang David Li | e6b8929 | 2016-04-18 17:47:38 +0000 | [diff] [blame] | 92 | return new InstrProfilingLegacyPass(Options); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 93 | } |
| 94 | |
Xinliang David Li | e6b8929 | 2016-04-18 17:47:38 +0000 | [diff] [blame] | 95 | bool InstrProfiling::isMachO() const { |
| 96 | return Triple(M->getTargetTriple()).isOSBinFormatMachO(); |
| 97 | } |
| 98 | |
| 99 | /// Get the section name for the counter variables. |
| 100 | StringRef InstrProfiling::getCountersSection() const { |
| 101 | return getInstrProfCountersSectionName(isMachO()); |
| 102 | } |
| 103 | |
| 104 | /// Get the section name for the name variables. |
| 105 | StringRef InstrProfiling::getNameSection() const { |
| 106 | return getInstrProfNameSectionName(isMachO()); |
| 107 | } |
| 108 | |
| 109 | /// Get the section name for the profile data variables. |
| 110 | StringRef InstrProfiling::getDataSection() const { |
| 111 | return getInstrProfDataSectionName(isMachO()); |
| 112 | } |
| 113 | |
| 114 | /// Get the section name for the coverage mapping data. |
| 115 | StringRef InstrProfiling::getCoverageSection() const { |
| 116 | return getInstrProfCoverageSectionName(isMachO()); |
| 117 | } |
| 118 | |
Xinliang David Li | 4ca1733 | 2016-09-18 18:34:07 +0000 | [diff] [blame] | 119 | static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) { |
| 120 | InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr); |
| 121 | if (Inc) |
| 122 | return Inc; |
| 123 | return dyn_cast<InstrProfIncrementInst>(Instr); |
| 124 | } |
| 125 | |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 126 | bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) { |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 127 | bool MadeChange = false; |
| 128 | |
| 129 | this->M = &M; |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 130 | this->TLI = &TLI; |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 131 | NamesVar = nullptr; |
| 132 | NamesSize = 0; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 133 | ProfileDataMap.clear(); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 134 | UsedVars.clear(); |
| 135 | |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 136 | // We did not know how many value sites there would be inside |
| 137 | // the instrumented function. This is counting the number of instrumented |
| 138 | // target value sites to enter it as field in the profile data variable. |
Rong Xu | 294572f | 2016-01-19 18:29:54 +0000 | [diff] [blame] | 139 | for (Function &F : M) { |
| 140 | InstrProfIncrementInst *FirstProfIncInst = nullptr; |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 141 | for (BasicBlock &BB : F) |
Rong Xu | 294572f | 2016-01-19 18:29:54 +0000 | [diff] [blame] | 142 | for (auto I = BB.begin(), E = BB.end(); I != E; I++) |
| 143 | if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I)) |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 144 | computeNumValueSiteCounts(Ind); |
Rong Xu | 294572f | 2016-01-19 18:29:54 +0000 | [diff] [blame] | 145 | else if (FirstProfIncInst == nullptr) |
| 146 | FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I); |
| 147 | |
| 148 | // Value profiling intrinsic lowering requires per-function profile data |
| 149 | // variable to be created first. |
| 150 | if (FirstProfIncInst != nullptr) |
| 151 | static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst)); |
| 152 | } |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 153 | |
| 154 | for (Function &F : M) |
| 155 | for (BasicBlock &BB : F) |
| 156 | for (auto I = BB.begin(), E = BB.end(); I != E;) { |
| 157 | auto Instr = I++; |
Xinliang David Li | 4ca1733 | 2016-09-18 18:34:07 +0000 | [diff] [blame] | 158 | InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr); |
| 159 | if (Inc) { |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 160 | lowerIncrement(Inc); |
| 161 | MadeChange = true; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 162 | } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) { |
| 163 | lowerValueProfileInst(Ind); |
| 164 | MadeChange = true; |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 165 | } |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 166 | } |
| 167 | |
Xinliang David Li | 8105607 | 2016-01-07 20:05:49 +0000 | [diff] [blame] | 168 | if (GlobalVariable *CoverageNamesVar = |
Xinliang David Li | 440cd70 | 2016-01-20 00:24:36 +0000 | [diff] [blame] | 169 | M.getNamedGlobal(getCoverageUnusedNamesVarName())) { |
Xinliang David Li | 8105607 | 2016-01-07 20:05:49 +0000 | [diff] [blame] | 170 | lowerCoverageData(CoverageNamesVar); |
Justin Bogner | d24e185 | 2015-02-11 02:52:44 +0000 | [diff] [blame] | 171 | MadeChange = true; |
| 172 | } |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 173 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 174 | if (!MadeChange) |
| 175 | return false; |
| 176 | |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 177 | emitVNodes(); |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 178 | emitNameData(); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 179 | emitRegistration(); |
| 180 | emitRuntimeHook(); |
| 181 | emitUses(); |
| 182 | emitInitialization(); |
| 183 | return true; |
| 184 | } |
| 185 | |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 186 | static Constant *getOrInsertValueProfilingCall(Module &M, |
| 187 | const TargetLibraryInfo &TLI) { |
Xinliang David Li | c767323 | 2015-11-22 00:22:07 +0000 | [diff] [blame] | 188 | LLVMContext &Ctx = M.getContext(); |
| 189 | auto *ReturnTy = Type::getVoidTy(M.getContext()); |
| 190 | Type *ParamTypes[] = { |
| 191 | #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType |
| 192 | #include "llvm/ProfileData/InstrProfData.inc" |
| 193 | }; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 194 | auto *ValueProfilingCallTy = |
Xinliang David Li | c767323 | 2015-11-22 00:22:07 +0000 | [diff] [blame] | 195 | FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false); |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 196 | Constant *Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(), |
| 197 | ValueProfilingCallTy); |
| 198 | if (Function *FunRes = dyn_cast<Function>(Res)) { |
| 199 | if (auto AK = TLI.getExtAttrForI32Param(false)) |
| 200 | FunRes->addAttribute(3, AK); |
| 201 | } |
| 202 | return Res; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 203 | } |
| 204 | |
| 205 | void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) { |
| 206 | |
| 207 | GlobalVariable *Name = Ind->getName(); |
| 208 | uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); |
| 209 | uint64_t Index = Ind->getIndex()->getZExtValue(); |
| 210 | auto It = ProfileDataMap.find(Name); |
| 211 | if (It == ProfileDataMap.end()) { |
| 212 | PerFunctionProfileData PD; |
| 213 | PD.NumValueSites[ValueKind] = Index + 1; |
| 214 | ProfileDataMap[Name] = PD; |
| 215 | } else if (It->second.NumValueSites[ValueKind] <= Index) |
| 216 | It->second.NumValueSites[ValueKind] = Index + 1; |
| 217 | } |
| 218 | |
| 219 | void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { |
| 220 | |
| 221 | GlobalVariable *Name = Ind->getName(); |
| 222 | auto It = ProfileDataMap.find(Name); |
| 223 | assert(It != ProfileDataMap.end() && It->second.DataVar && |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 224 | "value profiling detected in function with no counter incerement"); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 225 | |
| 226 | GlobalVariable *DataVar = It->second.DataVar; |
| 227 | uint64_t ValueKind = Ind->getValueKind()->getZExtValue(); |
| 228 | uint64_t Index = Ind->getIndex()->getZExtValue(); |
| 229 | for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind) |
| 230 | Index += It->second.NumValueSites[Kind]; |
| 231 | |
| 232 | IRBuilder<> Builder(Ind); |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 233 | Value *Args[3] = {Ind->getTargetValue(), |
| 234 | Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()), |
| 235 | Builder.getInt32(Index)}; |
Marcin Koscielnicki | 1c2bd1e | 2016-11-21 11:57:19 +0000 | [diff] [blame] | 236 | CallInst *Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), |
| 237 | Args); |
| 238 | if (auto AK = TLI->getExtAttrForI32Param(false)) |
| 239 | Call->addAttribute(3, AK); |
| 240 | Ind->replaceAllUsesWith(Call); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 241 | Ind->eraseFromParent(); |
| 242 | } |
| 243 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 244 | void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) { |
| 245 | GlobalVariable *Counters = getOrCreateRegionCounters(Inc); |
| 246 | |
Duncan P. N. Exon Smith | e82c286 | 2015-10-13 17:39:10 +0000 | [diff] [blame] | 247 | IRBuilder<> Builder(Inc); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 248 | uint64_t Index = Inc->getIndex()->getZExtValue(); |
Diego Novillo | b3029d2 | 2015-06-04 11:45:32 +0000 | [diff] [blame] | 249 | Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index); |
| 250 | Value *Count = Builder.CreateLoad(Addr, "pgocount"); |
Xinliang David Li | a754c47 | 2016-09-20 19:07:22 +0000 | [diff] [blame] | 251 | Count = Builder.CreateAdd(Count, Inc->getStep()); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 252 | Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr)); |
| 253 | Inc->eraseFromParent(); |
| 254 | } |
| 255 | |
Xinliang David Li | 8105607 | 2016-01-07 20:05:49 +0000 | [diff] [blame] | 256 | void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { |
Justin Bogner | d24e185 | 2015-02-11 02:52:44 +0000 | [diff] [blame] | 257 | |
Xinliang David Li | 8105607 | 2016-01-07 20:05:49 +0000 | [diff] [blame] | 258 | ConstantArray *Names = |
| 259 | cast<ConstantArray>(CoverageNamesVar->getInitializer()); |
| 260 | for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) { |
| 261 | Constant *NC = Names->getOperand(I); |
| 262 | Value *V = NC->stripPointerCasts(); |
Justin Bogner | d24e185 | 2015-02-11 02:52:44 +0000 | [diff] [blame] | 263 | assert(isa<GlobalVariable>(V) && "Missing reference to function name"); |
| 264 | GlobalVariable *Name = cast<GlobalVariable>(V); |
| 265 | |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 266 | Name->setLinkage(GlobalValue::PrivateLinkage); |
| 267 | ReferencedNames.push_back(Name); |
Justin Bogner | d24e185 | 2015-02-11 02:52:44 +0000 | [diff] [blame] | 268 | } |
| 269 | } |
| 270 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 271 | /// Get the name of a profiling variable for a particular function. |
Xinliang David Li | 83bc422 | 2015-10-22 20:32:12 +0000 | [diff] [blame] | 272 | static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) { |
Xinliang David Li | d1bab96 | 2015-12-12 17:28:03 +0000 | [diff] [blame] | 273 | StringRef NamePrefix = getInstrProfNameVarPrefix(); |
| 274 | StringRef Name = Inc->getName()->getName().substr(NamePrefix.size()); |
Xinliang David Li | 83bc422 | 2015-10-22 20:32:12 +0000 | [diff] [blame] | 275 | return (Prefix + Name).str(); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 276 | } |
| 277 | |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 278 | static inline bool shouldRecordFunctionAddr(Function *F) { |
| 279 | // Check the linkage |
| 280 | if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() && |
| 281 | !F->hasAvailableExternallyLinkage()) |
| 282 | return true; |
Rong Xu | af5aeba | 2016-04-27 21:17:30 +0000 | [diff] [blame] | 283 | // Prohibit function address recording if the function is both internal and |
| 284 | // COMDAT. This avoids the profile data variable referencing internal symbols |
| 285 | // in COMDAT. |
| 286 | if (F->hasLocalLinkage() && F->hasComdat()) |
| 287 | return false; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 288 | // Check uses of this function for other than direct calls or invokes to it. |
Xinliang David Li | 7008ce3 | 2016-06-02 16:33:41 +0000 | [diff] [blame] | 289 | // Inline virtual functions have linkeOnceODR linkage. When a key method |
| 290 | // exists, the vtable will only be emitted in the TU where the key method |
| 291 | // is defined. In a TU where vtable is not available, the function won't |
Xinliang David Li | 6c44e9e | 2016-06-03 23:02:28 +0000 | [diff] [blame] | 292 | // be 'addresstaken'. If its address is not recorded here, the profile data |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 293 | // with missing address may be picked by the linker leading to missing |
Xinliang David Li | 6c44e9e | 2016-06-03 23:02:28 +0000 | [diff] [blame] | 294 | // indirect call target info. |
| 295 | return F->hasAddressTaken() || F->hasLinkOnceLinkage(); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 296 | } |
| 297 | |
Xinliang David Li | 985ff20 | 2016-02-27 23:11:30 +0000 | [diff] [blame] | 298 | static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F, |
Xinliang David Li | ab361ef | 2015-12-21 21:52:27 +0000 | [diff] [blame] | 299 | InstrProfIncrementInst *Inc) { |
Xinliang David Li | 985ff20 | 2016-02-27 23:11:30 +0000 | [diff] [blame] | 300 | if (!needsComdatForCounter(F, M)) |
| 301 | return nullptr; |
| 302 | |
Xinliang David Li | ab361ef | 2015-12-21 21:52:27 +0000 | [diff] [blame] | 303 | // COFF format requires a COMDAT section to have a key symbol with the same |
Vedant Kumar | 2d5b5d3 | 2016-02-03 23:22:43 +0000 | [diff] [blame] | 304 | // name. The linker targeting COFF also requires that the COMDAT |
Xinliang David Li | 5fe0455 | 2015-12-22 00:11:15 +0000 | [diff] [blame] | 305 | // a section is associated to must precede the associating section. For this |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 306 | // reason, we must choose the counter var's name as the name of the comdat. |
Xinliang David Li | ab361ef | 2015-12-21 21:52:27 +0000 | [diff] [blame] | 307 | StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF() |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 308 | ? getInstrProfCountersVarPrefix() |
Xinliang David Li | ab361ef | 2015-12-21 21:52:27 +0000 | [diff] [blame] | 309 | : getInstrProfComdatPrefix()); |
| 310 | return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix))); |
| 311 | } |
| 312 | |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 313 | static bool needsRuntimeRegistrationOfSectionRange(const Module &M) { |
| 314 | // Don't do this for Darwin. compiler-rt uses linker magic. |
| 315 | if (Triple(M.getTargetTriple()).isOSDarwin()) |
| 316 | return false; |
| 317 | |
| 318 | // Use linker script magic to get data/cnts/name start/end. |
| 319 | if (Triple(M.getTargetTriple()).isOSLinux() || |
| 320 | Triple(M.getTargetTriple()).isOSFreeBSD() || |
| 321 | Triple(M.getTargetTriple()).isPS4CPU()) |
| 322 | return false; |
| 323 | |
| 324 | return true; |
| 325 | } |
| 326 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 327 | GlobalVariable * |
| 328 | InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 329 | GlobalVariable *NamePtr = Inc->getName(); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 330 | auto It = ProfileDataMap.find(NamePtr); |
| 331 | PerFunctionProfileData PD; |
| 332 | if (It != ProfileDataMap.end()) { |
| 333 | if (It->second.RegionCounters) |
| 334 | return It->second.RegionCounters; |
| 335 | PD = It->second; |
| 336 | } |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 337 | |
Wei Mi | 3cc9204 | 2015-09-23 22:40:45 +0000 | [diff] [blame] | 338 | // Move the name variable to the right section. Place them in a COMDAT group |
| 339 | // if the associated function is a COMDAT. This will make sure that |
| 340 | // only one copy of counters of the COMDAT function will be emitted after |
| 341 | // linking. |
Diego Novillo | df4837b | 2015-05-27 19:34:01 +0000 | [diff] [blame] | 342 | Function *Fn = Inc->getParent()->getParent(); |
Wei Mi | 3cc9204 | 2015-09-23 22:40:45 +0000 | [diff] [blame] | 343 | Comdat *ProfileVarsComdat = nullptr; |
Xinliang David Li | 985ff20 | 2016-02-27 23:11:30 +0000 | [diff] [blame] | 344 | ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 345 | |
| 346 | uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); |
| 347 | LLVMContext &Ctx = M->getContext(); |
| 348 | ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); |
| 349 | |
| 350 | // Create the counters variable. |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 351 | auto *CounterPtr = |
| 352 | new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(), |
Xinliang David Li | 83bc422 | 2015-10-22 20:32:12 +0000 | [diff] [blame] | 353 | Constant::getNullValue(CounterTy), |
| 354 | getVarName(Inc, getInstrProfCountersVarPrefix())); |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 355 | CounterPtr->setVisibility(NamePtr->getVisibility()); |
| 356 | CounterPtr->setSection(getCountersSection()); |
| 357 | CounterPtr->setAlignment(8); |
| 358 | CounterPtr->setComdat(ProfileVarsComdat); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 359 | |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 360 | auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 361 | // Allocate statically the array of pointers to value profile nodes for |
| 362 | // the current function. |
| 363 | Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy); |
| 364 | if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) { |
| 365 | |
| 366 | uint64_t NS = 0; |
| 367 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
| 368 | NS += PD.NumValueSites[Kind]; |
| 369 | if (NS) { |
| 370 | ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS); |
| 371 | |
| 372 | auto *ValuesVar = |
| 373 | new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(), |
| 374 | Constant::getNullValue(ValuesTy), |
| 375 | getVarName(Inc, getInstrProfValuesVarPrefix())); |
| 376 | ValuesVar->setVisibility(NamePtr->getVisibility()); |
| 377 | ValuesVar->setSection(getInstrProfValuesSectionName(isMachO())); |
| 378 | ValuesVar->setAlignment(8); |
| 379 | ValuesVar->setComdat(ProfileVarsComdat); |
| 380 | ValuesPtrExpr = |
| 381 | ConstantExpr::getBitCast(ValuesVar, llvm::Type::getInt8PtrTy(Ctx)); |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | // Create data variable. |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 386 | auto *Int16Ty = Type::getInt16Ty(Ctx); |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 387 | auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1); |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 388 | Type *DataTypes[] = { |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 389 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType, |
| 390 | #include "llvm/ProfileData/InstrProfData.inc" |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 391 | }; |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 392 | auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes)); |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 393 | |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 394 | Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) |
| 395 | ? ConstantExpr::getBitCast(Fn, Int8PtrTy) |
| 396 | : ConstantPointerNull::get(Int8PtrTy); |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 397 | |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 398 | Constant *Int16ArrayVals[IPVK_Last + 1]; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 399 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
| 400 | Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]); |
| 401 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 402 | Constant *DataVals[] = { |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 403 | #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, |
| 404 | #include "llvm/ProfileData/InstrProfData.inc" |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 405 | }; |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 406 | auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(), |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 407 | ConstantStruct::get(DataTy, DataVals), |
Xinliang David Li | 83bc422 | 2015-10-22 20:32:12 +0000 | [diff] [blame] | 408 | getVarName(Inc, getInstrProfDataVarPrefix())); |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 409 | Data->setVisibility(NamePtr->getVisibility()); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 410 | Data->setSection(getDataSection()); |
Xinliang David Li | c7c1f85 | 2015-11-23 18:02:59 +0000 | [diff] [blame] | 411 | Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT); |
Wei Mi | 3cc9204 | 2015-09-23 22:40:45 +0000 | [diff] [blame] | 412 | Data->setComdat(ProfileVarsComdat); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 413 | |
Betul Buyukkurt | 6fac174 | 2015-11-18 18:14:55 +0000 | [diff] [blame] | 414 | PD.RegionCounters = CounterPtr; |
| 415 | PD.DataVar = Data; |
| 416 | ProfileDataMap[NamePtr] = PD; |
| 417 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 418 | // Mark the data variable as used so that it isn't stripped out. |
| 419 | UsedVars.push_back(Data); |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 420 | // Now that the linkage set by the FE has been passed to the data and counter |
| 421 | // variables, reset Name variable's linkage and visibility to private so that |
| 422 | // it can be removed later by the compiler. |
| 423 | NamePtr->setLinkage(GlobalValue::PrivateLinkage); |
| 424 | // Collect the referenced names to be used by emitNameData. |
| 425 | ReferencedNames.push_back(NamePtr); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 426 | |
Xinliang David Li | 192c748 | 2015-11-05 00:47:26 +0000 | [diff] [blame] | 427 | return CounterPtr; |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 428 | } |
| 429 | |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 430 | void InstrProfiling::emitVNodes() { |
| 431 | if (!ValueProfileStaticAlloc) |
| 432 | return; |
Xinliang David Li | 8da773b | 2016-05-17 20:19:03 +0000 | [diff] [blame] | 433 | |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 434 | // For now only support this on platforms that do |
| 435 | // not require runtime registration to discover |
| 436 | // named section start/end. |
| 437 | if (needsRuntimeRegistrationOfSectionRange(*M)) |
| 438 | return; |
Xinliang David Li | 8da773b | 2016-05-17 20:19:03 +0000 | [diff] [blame] | 439 | |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 440 | size_t TotalNS = 0; |
| 441 | for (auto &PD : ProfileDataMap) { |
| 442 | for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) |
| 443 | TotalNS += PD.second.NumValueSites[Kind]; |
| 444 | } |
| 445 | |
| 446 | if (!TotalNS) |
| 447 | return; |
| 448 | |
| 449 | uint64_t NumCounters = TotalNS * NumCountersPerValueSite; |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 450 | // Heuristic for small programs with very few total value sites. |
| 451 | // The default value of vp-counters-per-site is chosen based on |
| 452 | // the observation that large apps usually have a low percentage |
| 453 | // of value sites that actually have any profile data, and thus |
| 454 | // the average number of counters per site is low. For small |
| 455 | // apps with very few sites, this may not be true. Bump up the |
| 456 | // number of counters in this case. |
Xinliang David Li | e452076 | 2016-05-23 19:29:26 +0000 | [diff] [blame] | 457 | #define INSTR_PROF_MIN_VAL_COUNTS 10 |
| 458 | if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS) |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 459 | NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2); |
Xinliang David Li | b628dd3 | 2016-05-21 22:55:34 +0000 | [diff] [blame] | 460 | |
| 461 | auto &Ctx = M->getContext(); |
| 462 | Type *VNodeTypes[] = { |
| 463 | #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType, |
| 464 | #include "llvm/ProfileData/InstrProfData.inc" |
| 465 | }; |
| 466 | auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes)); |
| 467 | |
| 468 | ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters); |
| 469 | auto *VNodesVar = new GlobalVariable( |
| 470 | *M, VNodesTy, false, llvm::GlobalValue::PrivateLinkage, |
| 471 | Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName()); |
| 472 | VNodesVar->setSection(getInstrProfVNodesSectionName(isMachO())); |
| 473 | UsedVars.push_back(VNodesVar); |
Xinliang David Li | 8da773b | 2016-05-17 20:19:03 +0000 | [diff] [blame] | 474 | } |
| 475 | |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 476 | void InstrProfiling::emitNameData() { |
| 477 | std::string UncompressedData; |
| 478 | |
| 479 | if (ReferencedNames.empty()) |
| 480 | return; |
| 481 | |
| 482 | std::string CompressedNameStr; |
Vedant Kumar | 9152fd1 | 2016-05-19 03:54:45 +0000 | [diff] [blame] | 483 | if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr, |
Vedant Kumar | 43cba73 | 2016-05-03 16:53:17 +0000 | [diff] [blame] | 484 | DoNameCompression)) { |
Vedant Kumar | 9152fd1 | 2016-05-19 03:54:45 +0000 | [diff] [blame] | 485 | llvm::report_fatal_error(toString(std::move(E)), false); |
Vedant Kumar | 43cba73 | 2016-05-03 16:53:17 +0000 | [diff] [blame] | 486 | } |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 487 | |
| 488 | auto &Ctx = M->getContext(); |
| 489 | auto *NamesVal = llvm::ConstantDataArray::getString( |
| 490 | Ctx, StringRef(CompressedNameStr), false); |
| 491 | NamesVar = new llvm::GlobalVariable(*M, NamesVal->getType(), true, |
| 492 | llvm::GlobalValue::PrivateLinkage, |
| 493 | NamesVal, getInstrProfNamesVarName()); |
| 494 | NamesSize = CompressedNameStr.size(); |
| 495 | NamesVar->setSection(getNameSection()); |
| 496 | UsedVars.push_back(NamesVar); |
| 497 | } |
| 498 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 499 | void InstrProfiling::emitRegistration() { |
Xinliang David Li | 8da773b | 2016-05-17 20:19:03 +0000 | [diff] [blame] | 500 | if (!needsRuntimeRegistrationOfSectionRange(*M)) |
Xinliang David Li | aa0592c | 2015-10-19 04:17:10 +0000 | [diff] [blame] | 501 | return; |
Xinliang David Li | 3dd8817 | 2015-10-13 18:39:48 +0000 | [diff] [blame] | 502 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 503 | // Construct the function. |
| 504 | auto *VoidTy = Type::getVoidTy(M->getContext()); |
| 505 | auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext()); |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 506 | auto *Int64Ty = Type::getInt64Ty(M->getContext()); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 507 | auto *RegisterFTy = FunctionType::get(VoidTy, false); |
| 508 | auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage, |
Xinliang David Li | 8ee08b0 | 2015-10-23 04:22:58 +0000 | [diff] [blame] | 509 | getInstrProfRegFuncsName(), M); |
Peter Collingbourne | 96efdd6 | 2016-06-14 21:01:22 +0000 | [diff] [blame] | 510 | RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 511 | if (Options.NoRedZone) |
| 512 | RegisterF->addFnAttr(Attribute::NoRedZone); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 513 | |
Diego Novillo | b3029d2 | 2015-06-04 11:45:32 +0000 | [diff] [blame] | 514 | auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 515 | auto *RuntimeRegisterF = |
| 516 | Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage, |
Xinliang David Li | 8ee08b0 | 2015-10-23 04:22:58 +0000 | [diff] [blame] | 517 | getInstrProfRegFuncName(), M); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 518 | |
| 519 | IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF)); |
| 520 | for (Value *Data : UsedVars) |
Xinliang David Li | a82d6c0 | 2016-02-08 18:13:49 +0000 | [diff] [blame] | 521 | if (Data != NamesVar) |
| 522 | IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy)); |
| 523 | |
| 524 | if (NamesVar) { |
| 525 | Type *ParamTypes[] = {VoidPtrTy, Int64Ty}; |
| 526 | auto *NamesRegisterTy = |
| 527 | FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false); |
| 528 | auto *NamesRegisterF = |
| 529 | Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage, |
| 530 | getInstrProfNamesRegFuncName(), M); |
| 531 | IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy), |
| 532 | IRB.getInt64(NamesSize)}); |
| 533 | } |
| 534 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 535 | IRB.CreateRetVoid(); |
| 536 | } |
| 537 | |
| 538 | void InstrProfiling::emitRuntimeHook() { |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 539 | |
Xinliang David Li | 7a88ad6 | 2015-10-29 04:08:31 +0000 | [diff] [blame] | 540 | // We expect the linker to be invoked with -u<hook_var> flag for linux, |
| 541 | // for which case there is no need to emit the user function. |
| 542 | if (Triple(M->getTargetTriple()).isOSLinux()) |
| 543 | return; |
| 544 | |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 545 | // If the module's provided its own runtime, we don't need to do anything. |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 546 | if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) |
| 547 | return; |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 548 | |
| 549 | // Declare an external variable that will pull in the runtime initialization. |
| 550 | auto *Int32Ty = Type::getInt32Ty(M->getContext()); |
| 551 | auto *Var = |
| 552 | new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage, |
Xinliang David Li | 8ee08b0 | 2015-10-23 04:22:58 +0000 | [diff] [blame] | 553 | nullptr, getInstrProfRuntimeHookVarName()); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 554 | |
| 555 | // Make a function that uses it. |
Xinliang David Li | 8ee08b0 | 2015-10-23 04:22:58 +0000 | [diff] [blame] | 556 | auto *User = Function::Create(FunctionType::get(Int32Ty, false), |
| 557 | GlobalValue::LinkOnceODRLinkage, |
| 558 | getInstrProfRuntimeHookVarUseFuncName(), M); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 559 | User->addFnAttr(Attribute::NoInline); |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 560 | if (Options.NoRedZone) |
| 561 | User->addFnAttr(Attribute::NoRedZone); |
Justin Bogner | 2e427d4 | 2015-02-25 22:52:20 +0000 | [diff] [blame] | 562 | User->setVisibility(GlobalValue::HiddenVisibility); |
Xinliang David Li | a228608 | 2016-05-25 17:17:51 +0000 | [diff] [blame] | 563 | if (Triple(M->getTargetTriple()).supportsCOMDAT()) |
Xinliang David Li | f4edae6 | 2016-05-24 18:47:38 +0000 | [diff] [blame] | 564 | User->setComdat(M->getOrInsertComdat(User->getName())); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 565 | |
| 566 | IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User)); |
| 567 | auto *Load = IRB.CreateLoad(Var); |
| 568 | IRB.CreateRet(Load); |
| 569 | |
| 570 | // Mark the user variable as used so that it isn't stripped out. |
| 571 | UsedVars.push_back(User); |
| 572 | } |
| 573 | |
| 574 | void InstrProfiling::emitUses() { |
Evgeniy Stepanov | ea6d49d | 2016-10-25 23:53:31 +0000 | [diff] [blame] | 575 | if (!UsedVars.empty()) |
| 576 | appendToUsed(*M, UsedVars); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 577 | } |
| 578 | |
| 579 | void InstrProfiling::emitInitialization() { |
Vedant Kumar | cd32eba | 2016-07-21 17:50:07 +0000 | [diff] [blame] | 580 | StringRef InstrProfileOutput = Options.InstrProfileOutput; |
Justin Bogner | ba1900c | 2015-04-30 23:49:23 +0000 | [diff] [blame] | 581 | |
Xinliang David Li | 6f8c504 | 2016-07-21 23:19:10 +0000 | [diff] [blame] | 582 | if (!InstrProfileOutput.empty()) { |
| 583 | // Create variable for profile name. |
| 584 | Constant *ProfileNameConst = |
| 585 | ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true); |
| 586 | GlobalVariable *ProfileNameVar = new GlobalVariable( |
| 587 | *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage, |
| 588 | ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)); |
| 589 | Triple TT(M->getTargetTriple()); |
| 590 | if (TT.supportsCOMDAT()) { |
| 591 | ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage); |
| 592 | ProfileNameVar->setComdat(M->getOrInsertComdat( |
| 593 | StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR)))); |
| 594 | } |
| 595 | } |
| 596 | |
Xinliang David Li | 8ee08b0 | 2015-10-23 04:22:58 +0000 | [diff] [blame] | 597 | Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName()); |
Xinliang David Li | 6f8c504 | 2016-07-21 23:19:10 +0000 | [diff] [blame] | 598 | if (!RegisterF) |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 599 | return; |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 600 | |
| 601 | // Create the initialization function. |
| 602 | auto *VoidTy = Type::getVoidTy(M->getContext()); |
Xinliang David Li | 8ee08b0 | 2015-10-23 04:22:58 +0000 | [diff] [blame] | 603 | auto *F = Function::Create(FunctionType::get(VoidTy, false), |
| 604 | GlobalValue::InternalLinkage, |
| 605 | getInstrProfInitFuncName(), M); |
Peter Collingbourne | 96efdd6 | 2016-06-14 21:01:22 +0000 | [diff] [blame] | 606 | F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 607 | F->addFnAttr(Attribute::NoInline); |
Xinliang David Li | 69a00f0 | 2016-06-21 02:39:08 +0000 | [diff] [blame] | 608 | if (Options.NoRedZone) |
| 609 | F->addFnAttr(Attribute::NoRedZone); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 610 | |
| 611 | // Add the basic block and the necessary calls. |
| 612 | IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F)); |
Justin Bogner | ba1900c | 2015-04-30 23:49:23 +0000 | [diff] [blame] | 613 | if (RegisterF) |
David Blaikie | ff6409d | 2015-05-18 22:13:54 +0000 | [diff] [blame] | 614 | IRB.CreateCall(RegisterF, {}); |
Justin Bogner | 61ba2e3 | 2014-12-08 18:02:35 +0000 | [diff] [blame] | 615 | IRB.CreateRetVoid(); |
| 616 | |
| 617 | appendToGlobalCtors(*M, F, 0); |
| 618 | } |