blob: 8da3e31200f3a9639418ce68ac0eb9aae390e37e [file] [log] [blame]
Justin Bogner61ba2e32014-12-08 18:02:35 +00001//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
Betul Buyukkurt6fac1742015-11-18 18:14:55 +000010// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
11// It also builds the data structures and initialization code needed for
12// updating execution counts and emitting the profile at runtime.
Justin Bogner61ba2e32014-12-08 18:02:35 +000013//
14//===----------------------------------------------------------------------===//
15
Xinliang David Li69a00f02016-06-21 02:39:08 +000016#include "llvm/Transforms/InstrProfiling.h"
Justin Bogner61ba2e32014-12-08 18:02:35 +000017#include "llvm/ADT/Triple.h"
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +000018#include "llvm/Analysis/TargetLibraryInfo.h"
Justin Bogner61ba2e32014-12-08 18:02:35 +000019#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/IntrinsicInst.h"
21#include "llvm/IR/Module.h"
Betul Buyukkurt6fac1742015-11-18 18:14:55 +000022#include "llvm/ProfileData/InstrProf.h"
Justin Bogner61ba2e32014-12-08 18:02:35 +000023#include "llvm/Transforms/Utils/ModuleUtils.h"
24
25using namespace llvm;
26
27#define DEBUG_TYPE "instrprof"
28
29namespace {
30
Xinliang David Lia82d6c02016-02-08 18:13:49 +000031cl::opt<bool> DoNameCompression("enable-name-compression",
32 cl::desc("Enable name string compression"),
33 cl::init(true));
34
Xinliang David Lib628dd32016-05-21 22:55:34 +000035cl::opt<bool> ValueProfileStaticAlloc(
36 "vp-static-alloc",
37 cl::desc("Do static counter allocation for value profiler"),
38 cl::init(true));
39cl::opt<double> NumCountersPerValueSite(
40 "vp-counters-per-site",
41 cl::desc("The average number of profile counters allocated "
42 "per value profiling site."),
43 // This is set to a very small value because in real programs, only
44 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
45 // For those sites with non-zero profile, the average number of targets
46 // is usually smaller than 2.
47 cl::init(1.0));
48
Xinliang David Lie6b89292016-04-18 17:47:38 +000049class InstrProfilingLegacyPass : public ModulePass {
50 InstrProfiling InstrProf;
51
Justin Bogner61ba2e32014-12-08 18:02:35 +000052public:
53 static char ID;
Xinliang David Lie6b89292016-04-18 17:47:38 +000054 InstrProfilingLegacyPass() : ModulePass(ID), InstrProf() {}
55 InstrProfilingLegacyPass(const InstrProfOptions &Options)
56 : ModulePass(ID), InstrProf(Options) {}
Mehdi Amini117296c2016-10-01 02:56:57 +000057 StringRef getPassName() const override {
Justin Bogner61ba2e32014-12-08 18:02:35 +000058 return "Frontend instrumentation-based coverage lowering";
59 }
60
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +000061 bool runOnModule(Module &M) override {
62 return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
63 }
Justin Bogner61ba2e32014-12-08 18:02:35 +000064
65 void getAnalysisUsage(AnalysisUsage &AU) const override {
66 AU.setPreservesCFG();
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +000067 AU.addRequired<TargetLibraryInfoWrapperPass>();
Justin Bogner61ba2e32014-12-08 18:02:35 +000068 }
Justin Bogner61ba2e32014-12-08 18:02:35 +000069};
70
71} // anonymous namespace
72
Sean Silvafd03ac62016-08-09 00:28:38 +000073PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +000074 auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
75 if (!run(M, TLI))
Xinliang David Lie6b89292016-04-18 17:47:38 +000076 return PreservedAnalyses::all();
77
78 return PreservedAnalyses::none();
79}
80
81char InstrProfilingLegacyPass::ID = 0;
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +000082INITIALIZE_PASS_BEGIN(
83 InstrProfilingLegacyPass, "instrprof",
84 "Frontend instrumentation-based coverage lowering.", false, false)
85INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
86INITIALIZE_PASS_END(
87 InstrProfilingLegacyPass, "instrprof",
88 "Frontend instrumentation-based coverage lowering.", false, false)
Justin Bogner61ba2e32014-12-08 18:02:35 +000089
Xinliang David Li69a00f02016-06-21 02:39:08 +000090ModulePass *
91llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
Xinliang David Lie6b89292016-04-18 17:47:38 +000092 return new InstrProfilingLegacyPass(Options);
Justin Bogner61ba2e32014-12-08 18:02:35 +000093}
94
Xinliang David Lie6b89292016-04-18 17:47:38 +000095bool InstrProfiling::isMachO() const {
96 return Triple(M->getTargetTriple()).isOSBinFormatMachO();
97}
98
99/// Get the section name for the counter variables.
100StringRef InstrProfiling::getCountersSection() const {
101 return getInstrProfCountersSectionName(isMachO());
102}
103
104/// Get the section name for the name variables.
105StringRef InstrProfiling::getNameSection() const {
106 return getInstrProfNameSectionName(isMachO());
107}
108
109/// Get the section name for the profile data variables.
110StringRef InstrProfiling::getDataSection() const {
111 return getInstrProfDataSectionName(isMachO());
112}
113
114/// Get the section name for the coverage mapping data.
115StringRef InstrProfiling::getCoverageSection() const {
116 return getInstrProfCoverageSectionName(isMachO());
117}
118
Xinliang David Li4ca17332016-09-18 18:34:07 +0000119static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
120 InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
121 if (Inc)
122 return Inc;
123 return dyn_cast<InstrProfIncrementInst>(Instr);
124}
125
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +0000126bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
Justin Bogner61ba2e32014-12-08 18:02:35 +0000127 bool MadeChange = false;
128
129 this->M = &M;
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +0000130 this->TLI = &TLI;
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000131 NamesVar = nullptr;
132 NamesSize = 0;
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000133 ProfileDataMap.clear();
Justin Bogner61ba2e32014-12-08 18:02:35 +0000134 UsedVars.clear();
135
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000136 // We did not know how many value sites there would be inside
137 // the instrumented function. This is counting the number of instrumented
138 // target value sites to enter it as field in the profile data variable.
Rong Xu294572f2016-01-19 18:29:54 +0000139 for (Function &F : M) {
140 InstrProfIncrementInst *FirstProfIncInst = nullptr;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000141 for (BasicBlock &BB : F)
Rong Xu294572f2016-01-19 18:29:54 +0000142 for (auto I = BB.begin(), E = BB.end(); I != E; I++)
143 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000144 computeNumValueSiteCounts(Ind);
Rong Xu294572f2016-01-19 18:29:54 +0000145 else if (FirstProfIncInst == nullptr)
146 FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
147
148 // Value profiling intrinsic lowering requires per-function profile data
149 // variable to be created first.
150 if (FirstProfIncInst != nullptr)
151 static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
152 }
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000153
154 for (Function &F : M)
155 for (BasicBlock &BB : F)
156 for (auto I = BB.begin(), E = BB.end(); I != E;) {
157 auto Instr = I++;
Xinliang David Li4ca17332016-09-18 18:34:07 +0000158 InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
159 if (Inc) {
Justin Bogner61ba2e32014-12-08 18:02:35 +0000160 lowerIncrement(Inc);
161 MadeChange = true;
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000162 } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
163 lowerValueProfileInst(Ind);
164 MadeChange = true;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000165 }
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000166 }
167
Xinliang David Li81056072016-01-07 20:05:49 +0000168 if (GlobalVariable *CoverageNamesVar =
Xinliang David Li440cd702016-01-20 00:24:36 +0000169 M.getNamedGlobal(getCoverageUnusedNamesVarName())) {
Xinliang David Li81056072016-01-07 20:05:49 +0000170 lowerCoverageData(CoverageNamesVar);
Justin Bognerd24e1852015-02-11 02:52:44 +0000171 MadeChange = true;
172 }
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000173
Justin Bogner61ba2e32014-12-08 18:02:35 +0000174 if (!MadeChange)
175 return false;
176
Xinliang David Lib628dd32016-05-21 22:55:34 +0000177 emitVNodes();
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000178 emitNameData();
Justin Bogner61ba2e32014-12-08 18:02:35 +0000179 emitRegistration();
180 emitRuntimeHook();
181 emitUses();
182 emitInitialization();
183 return true;
184}
185
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +0000186static Constant *getOrInsertValueProfilingCall(Module &M,
187 const TargetLibraryInfo &TLI) {
Xinliang David Lic7673232015-11-22 00:22:07 +0000188 LLVMContext &Ctx = M.getContext();
189 auto *ReturnTy = Type::getVoidTy(M.getContext());
190 Type *ParamTypes[] = {
191#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
192#include "llvm/ProfileData/InstrProfData.inc"
193 };
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000194 auto *ValueProfilingCallTy =
Xinliang David Lic7673232015-11-22 00:22:07 +0000195 FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +0000196 Constant *Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
197 ValueProfilingCallTy);
198 if (Function *FunRes = dyn_cast<Function>(Res)) {
199 if (auto AK = TLI.getExtAttrForI32Param(false))
200 FunRes->addAttribute(3, AK);
201 }
202 return Res;
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000203}
204
205void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
206
207 GlobalVariable *Name = Ind->getName();
208 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
209 uint64_t Index = Ind->getIndex()->getZExtValue();
210 auto It = ProfileDataMap.find(Name);
211 if (It == ProfileDataMap.end()) {
212 PerFunctionProfileData PD;
213 PD.NumValueSites[ValueKind] = Index + 1;
214 ProfileDataMap[Name] = PD;
215 } else if (It->second.NumValueSites[ValueKind] <= Index)
216 It->second.NumValueSites[ValueKind] = Index + 1;
217}
218
219void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
220
221 GlobalVariable *Name = Ind->getName();
222 auto It = ProfileDataMap.find(Name);
223 assert(It != ProfileDataMap.end() && It->second.DataVar &&
Xinliang David Li69a00f02016-06-21 02:39:08 +0000224 "value profiling detected in function with no counter incerement");
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000225
226 GlobalVariable *DataVar = It->second.DataVar;
227 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
228 uint64_t Index = Ind->getIndex()->getZExtValue();
229 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
230 Index += It->second.NumValueSites[Kind];
231
232 IRBuilder<> Builder(Ind);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000233 Value *Args[3] = {Ind->getTargetValue(),
234 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
235 Builder.getInt32(Index)};
Marcin Koscielnicki1c2bd1e2016-11-21 11:57:19 +0000236 CallInst *Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI),
237 Args);
238 if (auto AK = TLI->getExtAttrForI32Param(false))
239 Call->addAttribute(3, AK);
240 Ind->replaceAllUsesWith(Call);
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000241 Ind->eraseFromParent();
242}
243
Justin Bogner61ba2e32014-12-08 18:02:35 +0000244void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
245 GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
246
Duncan P. N. Exon Smithe82c2862015-10-13 17:39:10 +0000247 IRBuilder<> Builder(Inc);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000248 uint64_t Index = Inc->getIndex()->getZExtValue();
Diego Novillob3029d22015-06-04 11:45:32 +0000249 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
250 Value *Count = Builder.CreateLoad(Addr, "pgocount");
Xinliang David Lia754c472016-09-20 19:07:22 +0000251 Count = Builder.CreateAdd(Count, Inc->getStep());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000252 Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
253 Inc->eraseFromParent();
254}
255
Xinliang David Li81056072016-01-07 20:05:49 +0000256void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
Justin Bognerd24e1852015-02-11 02:52:44 +0000257
Xinliang David Li81056072016-01-07 20:05:49 +0000258 ConstantArray *Names =
259 cast<ConstantArray>(CoverageNamesVar->getInitializer());
260 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
261 Constant *NC = Names->getOperand(I);
262 Value *V = NC->stripPointerCasts();
Justin Bognerd24e1852015-02-11 02:52:44 +0000263 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
264 GlobalVariable *Name = cast<GlobalVariable>(V);
265
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000266 Name->setLinkage(GlobalValue::PrivateLinkage);
267 ReferencedNames.push_back(Name);
Justin Bognerd24e1852015-02-11 02:52:44 +0000268 }
269}
270
Justin Bogner61ba2e32014-12-08 18:02:35 +0000271/// Get the name of a profiling variable for a particular function.
Xinliang David Li83bc4222015-10-22 20:32:12 +0000272static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
Xinliang David Lid1bab962015-12-12 17:28:03 +0000273 StringRef NamePrefix = getInstrProfNameVarPrefix();
274 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
Xinliang David Li83bc4222015-10-22 20:32:12 +0000275 return (Prefix + Name).str();
Justin Bogner61ba2e32014-12-08 18:02:35 +0000276}
277
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000278static inline bool shouldRecordFunctionAddr(Function *F) {
279 // Check the linkage
280 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
281 !F->hasAvailableExternallyLinkage())
282 return true;
Rong Xuaf5aeba2016-04-27 21:17:30 +0000283 // Prohibit function address recording if the function is both internal and
284 // COMDAT. This avoids the profile data variable referencing internal symbols
285 // in COMDAT.
286 if (F->hasLocalLinkage() && F->hasComdat())
287 return false;
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000288 // Check uses of this function for other than direct calls or invokes to it.
Xinliang David Li7008ce32016-06-02 16:33:41 +0000289 // Inline virtual functions have linkeOnceODR linkage. When a key method
290 // exists, the vtable will only be emitted in the TU where the key method
291 // is defined. In a TU where vtable is not available, the function won't
Xinliang David Li6c44e9e2016-06-03 23:02:28 +0000292 // be 'addresstaken'. If its address is not recorded here, the profile data
Xinliang David Li69a00f02016-06-21 02:39:08 +0000293 // with missing address may be picked by the linker leading to missing
Xinliang David Li6c44e9e2016-06-03 23:02:28 +0000294 // indirect call target info.
295 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000296}
297
Xinliang David Li985ff202016-02-27 23:11:30 +0000298static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F,
Xinliang David Liab361ef2015-12-21 21:52:27 +0000299 InstrProfIncrementInst *Inc) {
Xinliang David Li985ff202016-02-27 23:11:30 +0000300 if (!needsComdatForCounter(F, M))
301 return nullptr;
302
Xinliang David Liab361ef2015-12-21 21:52:27 +0000303 // COFF format requires a COMDAT section to have a key symbol with the same
Vedant Kumar2d5b5d32016-02-03 23:22:43 +0000304 // name. The linker targeting COFF also requires that the COMDAT
Xinliang David Li5fe04552015-12-22 00:11:15 +0000305 // a section is associated to must precede the associating section. For this
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000306 // reason, we must choose the counter var's name as the name of the comdat.
Xinliang David Liab361ef2015-12-21 21:52:27 +0000307 StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000308 ? getInstrProfCountersVarPrefix()
Xinliang David Liab361ef2015-12-21 21:52:27 +0000309 : getInstrProfComdatPrefix());
310 return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
311}
312
Xinliang David Lib628dd32016-05-21 22:55:34 +0000313static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
314 // Don't do this for Darwin. compiler-rt uses linker magic.
315 if (Triple(M.getTargetTriple()).isOSDarwin())
316 return false;
317
318 // Use linker script magic to get data/cnts/name start/end.
319 if (Triple(M.getTargetTriple()).isOSLinux() ||
320 Triple(M.getTargetTriple()).isOSFreeBSD() ||
321 Triple(M.getTargetTriple()).isPS4CPU())
322 return false;
323
324 return true;
325}
326
Justin Bogner61ba2e32014-12-08 18:02:35 +0000327GlobalVariable *
328InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
Xinliang David Li192c7482015-11-05 00:47:26 +0000329 GlobalVariable *NamePtr = Inc->getName();
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000330 auto It = ProfileDataMap.find(NamePtr);
331 PerFunctionProfileData PD;
332 if (It != ProfileDataMap.end()) {
333 if (It->second.RegionCounters)
334 return It->second.RegionCounters;
335 PD = It->second;
336 }
Justin Bogner61ba2e32014-12-08 18:02:35 +0000337
Wei Mi3cc92042015-09-23 22:40:45 +0000338 // Move the name variable to the right section. Place them in a COMDAT group
339 // if the associated function is a COMDAT. This will make sure that
340 // only one copy of counters of the COMDAT function will be emitted after
341 // linking.
Diego Novillodf4837b2015-05-27 19:34:01 +0000342 Function *Fn = Inc->getParent()->getParent();
Wei Mi3cc92042015-09-23 22:40:45 +0000343 Comdat *ProfileVarsComdat = nullptr;
Xinliang David Li985ff202016-02-27 23:11:30 +0000344 ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000345
346 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
347 LLVMContext &Ctx = M->getContext();
348 ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
349
350 // Create the counters variable.
Xinliang David Li192c7482015-11-05 00:47:26 +0000351 auto *CounterPtr =
352 new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
Xinliang David Li83bc4222015-10-22 20:32:12 +0000353 Constant::getNullValue(CounterTy),
354 getVarName(Inc, getInstrProfCountersVarPrefix()));
Xinliang David Li192c7482015-11-05 00:47:26 +0000355 CounterPtr->setVisibility(NamePtr->getVisibility());
356 CounterPtr->setSection(getCountersSection());
357 CounterPtr->setAlignment(8);
358 CounterPtr->setComdat(ProfileVarsComdat);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000359
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000360 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
Xinliang David Lib628dd32016-05-21 22:55:34 +0000361 // Allocate statically the array of pointers to value profile nodes for
362 // the current function.
363 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
364 if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
365
366 uint64_t NS = 0;
367 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
368 NS += PD.NumValueSites[Kind];
369 if (NS) {
370 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
371
372 auto *ValuesVar =
373 new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
374 Constant::getNullValue(ValuesTy),
375 getVarName(Inc, getInstrProfValuesVarPrefix()));
376 ValuesVar->setVisibility(NamePtr->getVisibility());
377 ValuesVar->setSection(getInstrProfValuesSectionName(isMachO()));
378 ValuesVar->setAlignment(8);
379 ValuesVar->setComdat(ProfileVarsComdat);
380 ValuesPtrExpr =
381 ConstantExpr::getBitCast(ValuesVar, llvm::Type::getInt8PtrTy(Ctx));
382 }
383 }
384
385 // Create data variable.
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000386 auto *Int16Ty = Type::getInt16Ty(Ctx);
Xinliang David Lib628dd32016-05-21 22:55:34 +0000387 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
Xinliang David Li192c7482015-11-05 00:47:26 +0000388 Type *DataTypes[] = {
Xinliang David Li69a00f02016-06-21 02:39:08 +0000389#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
390#include "llvm/ProfileData/InstrProfData.inc"
Xinliang David Li192c7482015-11-05 00:47:26 +0000391 };
Justin Bogner61ba2e32014-12-08 18:02:35 +0000392 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
Xinliang David Li192c7482015-11-05 00:47:26 +0000393
Xinliang David Li69a00f02016-06-21 02:39:08 +0000394 Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
395 ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
396 : ConstantPointerNull::get(Int8PtrTy);
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000397
Xinliang David Li69a00f02016-06-21 02:39:08 +0000398 Constant *Int16ArrayVals[IPVK_Last + 1];
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000399 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
400 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
401
Justin Bogner61ba2e32014-12-08 18:02:35 +0000402 Constant *DataVals[] = {
Xinliang David Li69a00f02016-06-21 02:39:08 +0000403#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
404#include "llvm/ProfileData/InstrProfData.inc"
Xinliang David Li192c7482015-11-05 00:47:26 +0000405 };
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000406 auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
Justin Bogner61ba2e32014-12-08 18:02:35 +0000407 ConstantStruct::get(DataTy, DataVals),
Xinliang David Li83bc4222015-10-22 20:32:12 +0000408 getVarName(Inc, getInstrProfDataVarPrefix()));
Xinliang David Li192c7482015-11-05 00:47:26 +0000409 Data->setVisibility(NamePtr->getVisibility());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000410 Data->setSection(getDataSection());
Xinliang David Lic7c1f852015-11-23 18:02:59 +0000411 Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
Wei Mi3cc92042015-09-23 22:40:45 +0000412 Data->setComdat(ProfileVarsComdat);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000413
Betul Buyukkurt6fac1742015-11-18 18:14:55 +0000414 PD.RegionCounters = CounterPtr;
415 PD.DataVar = Data;
416 ProfileDataMap[NamePtr] = PD;
417
Justin Bogner61ba2e32014-12-08 18:02:35 +0000418 // Mark the data variable as used so that it isn't stripped out.
419 UsedVars.push_back(Data);
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000420 // Now that the linkage set by the FE has been passed to the data and counter
421 // variables, reset Name variable's linkage and visibility to private so that
422 // it can be removed later by the compiler.
423 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
424 // Collect the referenced names to be used by emitNameData.
425 ReferencedNames.push_back(NamePtr);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000426
Xinliang David Li192c7482015-11-05 00:47:26 +0000427 return CounterPtr;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000428}
429
Xinliang David Lib628dd32016-05-21 22:55:34 +0000430void InstrProfiling::emitVNodes() {
431 if (!ValueProfileStaticAlloc)
432 return;
Xinliang David Li8da773b2016-05-17 20:19:03 +0000433
Xinliang David Lib628dd32016-05-21 22:55:34 +0000434 // For now only support this on platforms that do
435 // not require runtime registration to discover
436 // named section start/end.
437 if (needsRuntimeRegistrationOfSectionRange(*M))
438 return;
Xinliang David Li8da773b2016-05-17 20:19:03 +0000439
Xinliang David Lib628dd32016-05-21 22:55:34 +0000440 size_t TotalNS = 0;
441 for (auto &PD : ProfileDataMap) {
442 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
443 TotalNS += PD.second.NumValueSites[Kind];
444 }
445
446 if (!TotalNS)
447 return;
448
449 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
Xinliang David Li69a00f02016-06-21 02:39:08 +0000450// Heuristic for small programs with very few total value sites.
451// The default value of vp-counters-per-site is chosen based on
452// the observation that large apps usually have a low percentage
453// of value sites that actually have any profile data, and thus
454// the average number of counters per site is low. For small
455// apps with very few sites, this may not be true. Bump up the
456// number of counters in this case.
Xinliang David Lie4520762016-05-23 19:29:26 +0000457#define INSTR_PROF_MIN_VAL_COUNTS 10
458 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
Xinliang David Li69a00f02016-06-21 02:39:08 +0000459 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
Xinliang David Lib628dd32016-05-21 22:55:34 +0000460
461 auto &Ctx = M->getContext();
462 Type *VNodeTypes[] = {
463#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
464#include "llvm/ProfileData/InstrProfData.inc"
465 };
466 auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
467
468 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
469 auto *VNodesVar = new GlobalVariable(
470 *M, VNodesTy, false, llvm::GlobalValue::PrivateLinkage,
471 Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
472 VNodesVar->setSection(getInstrProfVNodesSectionName(isMachO()));
473 UsedVars.push_back(VNodesVar);
Xinliang David Li8da773b2016-05-17 20:19:03 +0000474}
475
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000476void InstrProfiling::emitNameData() {
477 std::string UncompressedData;
478
479 if (ReferencedNames.empty())
480 return;
481
482 std::string CompressedNameStr;
Vedant Kumar9152fd12016-05-19 03:54:45 +0000483 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
Vedant Kumar43cba732016-05-03 16:53:17 +0000484 DoNameCompression)) {
Vedant Kumar9152fd12016-05-19 03:54:45 +0000485 llvm::report_fatal_error(toString(std::move(E)), false);
Vedant Kumar43cba732016-05-03 16:53:17 +0000486 }
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000487
488 auto &Ctx = M->getContext();
489 auto *NamesVal = llvm::ConstantDataArray::getString(
490 Ctx, StringRef(CompressedNameStr), false);
491 NamesVar = new llvm::GlobalVariable(*M, NamesVal->getType(), true,
492 llvm::GlobalValue::PrivateLinkage,
493 NamesVal, getInstrProfNamesVarName());
494 NamesSize = CompressedNameStr.size();
495 NamesVar->setSection(getNameSection());
496 UsedVars.push_back(NamesVar);
497}
498
Justin Bogner61ba2e32014-12-08 18:02:35 +0000499void InstrProfiling::emitRegistration() {
Xinliang David Li8da773b2016-05-17 20:19:03 +0000500 if (!needsRuntimeRegistrationOfSectionRange(*M))
Xinliang David Liaa0592c2015-10-19 04:17:10 +0000501 return;
Xinliang David Li3dd88172015-10-13 18:39:48 +0000502
Justin Bogner61ba2e32014-12-08 18:02:35 +0000503 // Construct the function.
504 auto *VoidTy = Type::getVoidTy(M->getContext());
505 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000506 auto *Int64Ty = Type::getInt64Ty(M->getContext());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000507 auto *RegisterFTy = FunctionType::get(VoidTy, false);
508 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000509 getInstrProfRegFuncsName(), M);
Peter Collingbourne96efdd62016-06-14 21:01:22 +0000510 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000511 if (Options.NoRedZone)
512 RegisterF->addFnAttr(Attribute::NoRedZone);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000513
Diego Novillob3029d22015-06-04 11:45:32 +0000514 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000515 auto *RuntimeRegisterF =
516 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000517 getInstrProfRegFuncName(), M);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000518
519 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
520 for (Value *Data : UsedVars)
Xinliang David Lia82d6c02016-02-08 18:13:49 +0000521 if (Data != NamesVar)
522 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
523
524 if (NamesVar) {
525 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
526 auto *NamesRegisterTy =
527 FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
528 auto *NamesRegisterF =
529 Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
530 getInstrProfNamesRegFuncName(), M);
531 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
532 IRB.getInt64(NamesSize)});
533 }
534
Justin Bogner61ba2e32014-12-08 18:02:35 +0000535 IRB.CreateRetVoid();
536}
537
538void InstrProfiling::emitRuntimeHook() {
Justin Bogner61ba2e32014-12-08 18:02:35 +0000539
Xinliang David Li7a88ad62015-10-29 04:08:31 +0000540 // We expect the linker to be invoked with -u<hook_var> flag for linux,
541 // for which case there is no need to emit the user function.
542 if (Triple(M->getTargetTriple()).isOSLinux())
543 return;
544
Justin Bogner61ba2e32014-12-08 18:02:35 +0000545 // If the module's provided its own runtime, we don't need to do anything.
Xinliang David Li69a00f02016-06-21 02:39:08 +0000546 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
547 return;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000548
549 // Declare an external variable that will pull in the runtime initialization.
550 auto *Int32Ty = Type::getInt32Ty(M->getContext());
551 auto *Var =
552 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000553 nullptr, getInstrProfRuntimeHookVarName());
Justin Bogner61ba2e32014-12-08 18:02:35 +0000554
555 // Make a function that uses it.
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000556 auto *User = Function::Create(FunctionType::get(Int32Ty, false),
557 GlobalValue::LinkOnceODRLinkage,
558 getInstrProfRuntimeHookVarUseFuncName(), M);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000559 User->addFnAttr(Attribute::NoInline);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000560 if (Options.NoRedZone)
561 User->addFnAttr(Attribute::NoRedZone);
Justin Bogner2e427d42015-02-25 22:52:20 +0000562 User->setVisibility(GlobalValue::HiddenVisibility);
Xinliang David Lia2286082016-05-25 17:17:51 +0000563 if (Triple(M->getTargetTriple()).supportsCOMDAT())
Xinliang David Lif4edae62016-05-24 18:47:38 +0000564 User->setComdat(M->getOrInsertComdat(User->getName()));
Justin Bogner61ba2e32014-12-08 18:02:35 +0000565
566 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
567 auto *Load = IRB.CreateLoad(Var);
568 IRB.CreateRet(Load);
569
570 // Mark the user variable as used so that it isn't stripped out.
571 UsedVars.push_back(User);
572}
573
574void InstrProfiling::emitUses() {
Evgeniy Stepanovea6d49d2016-10-25 23:53:31 +0000575 if (!UsedVars.empty())
576 appendToUsed(*M, UsedVars);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000577}
578
579void InstrProfiling::emitInitialization() {
Vedant Kumarcd32eba2016-07-21 17:50:07 +0000580 StringRef InstrProfileOutput = Options.InstrProfileOutput;
Justin Bognerba1900c2015-04-30 23:49:23 +0000581
Xinliang David Li6f8c5042016-07-21 23:19:10 +0000582 if (!InstrProfileOutput.empty()) {
583 // Create variable for profile name.
584 Constant *ProfileNameConst =
585 ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
586 GlobalVariable *ProfileNameVar = new GlobalVariable(
587 *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
588 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
589 Triple TT(M->getTargetTriple());
590 if (TT.supportsCOMDAT()) {
591 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
592 ProfileNameVar->setComdat(M->getOrInsertComdat(
593 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
594 }
595 }
596
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000597 Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
Xinliang David Li6f8c5042016-07-21 23:19:10 +0000598 if (!RegisterF)
Xinliang David Li69a00f02016-06-21 02:39:08 +0000599 return;
Justin Bogner61ba2e32014-12-08 18:02:35 +0000600
601 // Create the initialization function.
602 auto *VoidTy = Type::getVoidTy(M->getContext());
Xinliang David Li8ee08b02015-10-23 04:22:58 +0000603 auto *F = Function::Create(FunctionType::get(VoidTy, false),
604 GlobalValue::InternalLinkage,
605 getInstrProfInitFuncName(), M);
Peter Collingbourne96efdd62016-06-14 21:01:22 +0000606 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000607 F->addFnAttr(Attribute::NoInline);
Xinliang David Li69a00f02016-06-21 02:39:08 +0000608 if (Options.NoRedZone)
609 F->addFnAttr(Attribute::NoRedZone);
Justin Bogner61ba2e32014-12-08 18:02:35 +0000610
611 // Add the basic block and the necessary calls.
612 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
Justin Bognerba1900c2015-04-30 23:49:23 +0000613 if (RegisterF)
David Blaikieff6409d2015-05-18 22:13:54 +0000614 IRB.CreateCall(RegisterF, {});
Justin Bogner61ba2e32014-12-08 18:02:35 +0000615 IRB.CreateRetVoid();
616
617 appendToGlobalCtors(*M, F, 0);
618}